From: Frank Lahm Date: Thu, 3 Jan 2013 06:31:27 +0000 (+0100) Subject: Add UTF8 support X-Git-Url: https://arthur.barton.de/cgi-bin/gitweb.cgi?p=netatalk.git;a=commitdiff_plain;h=1f756c9c1e9133e520e3cb7b0149e2c8445da791 Add UTF8 support --- diff --git a/etc/spotlight/spotlight_rawquery_lexer.c b/etc/spotlight/spotlight_rawquery_lexer.c index c278fba6..350df439 100644 --- a/etc/spotlight/spotlight_rawquery_lexer.c +++ b/etc/spotlight/spotlight_rawquery_lexer.c @@ -374,13 +374,14 @@ struct yy_trans_info flex_int32_t yy_verify; flex_int32_t yy_nxt; }; -static yyconst flex_int16_t yy_accept[48] = +static yyconst flex_int16_t yy_accept[57] = { 0, - 0, 0, 18, 17, 16, 17, 6, 17, 17, 7, - 8, 5, 15, 13, 17, 14, 5, 5, 5, 17, - 12, 0, 9, 5, 11, 5, 5, 5, 10, 0, - 5, 5, 5, 0, 5, 5, 4, 0, 5, 3, - 0, 5, 0, 1, 0, 2, 0 + 0, 0, 18, 17, 16, 17, 5, 17, 17, 6, + 7, 15, 14, 12, 17, 13, 15, 15, 15, 17, + 17, 17, 17, 11, 0, 8, 15, 0, 0, 0, + 10, 15, 15, 15, 9, 0, 0, 0, 15, 15, + 15, 0, 0, 15, 15, 4, 0, 15, 3, 0, + 15, 0, 1, 0, 2, 0 } ; static yyconst flex_int32_t yy_ec[256] = @@ -398,72 +399,85 @@ static yyconst flex_int32_t yy_ec[256] = 18, 19, 20, 9, 21, 9, 9, 22, 23, 24, 25, 9, 9, 26, 27, 28, 29, 9, 9, 9, - 9, 9, 1, 30, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 9, 9, 1, 30, 1, 1, 1, 31, 31, 31, + 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, + 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, + 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, + 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, + 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, + 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, + 31, 1, 1, 32, 32, 32, 32, 32, 32, 32, + + 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, + 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, + 32, 32, 32, 33, 33, 33, 33, 33, 33, 33, + 33, 33, 33, 33, 33, 33, 33, 33, 33, 34, + 34, 34, 34, 34, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 } ; -static yyconst flex_int32_t yy_meta[31] = +static yyconst flex_int32_t yy_meta[35] = { 0, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 2, 1, 1, 1, 2, 2, 2, 2, 2, 2, - 2, 2, 2, 2, 2, 2, 2, 2, 2, 1 + 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, + 1, 2, 2, 2 } ; -static yyconst flex_int16_t yy_base[49] = +static yyconst flex_int16_t yy_base[58] = { 0, - 0, 0, 57, 58, 58, 43, 58, 27, 48, 58, - 58, 0, 58, 58, 40, 58, 28, 34, 24, 19, - 58, 27, 58, 0, 58, 31, 24, 16, 58, 21, - 26, 15, 23, 22, 15, 20, 0, 26, 16, 0, - 14, 16, 6, 0, 7, 58, 58, 29 + 0, 0, 91, 92, 92, 77, 92, 61, 82, 92, + 92, 3, 92, 92, 74, 92, 14, 25, 15, 56, + 54, 53, 52, 92, 61, 92, 57, 49, 48, 47, + 92, 27, 28, 16, 92, 46, 45, 52, 36, 30, + 37, 43, 55, 48, 38, 47, 59, 39, 45, 47, + 40, 40, 42, 40, 92, 92, 42 } ; -static yyconst flex_int16_t yy_def[49] = +static yyconst flex_int16_t yy_def[58] = { 0, - 47, 1, 47, 47, 47, 47, 47, 47, 47, 47, - 47, 48, 47, 47, 47, 47, 48, 48, 48, 47, - 47, 47, 47, 48, 47, 48, 48, 48, 47, 47, - 48, 48, 48, 47, 48, 48, 48, 47, 48, 48, - 47, 48, 47, 48, 47, 47, 0, 47 + 56, 1, 56, 56, 56, 56, 56, 56, 56, 56, + 56, 57, 56, 56, 56, 56, 57, 17, 17, 56, + 56, 56, 56, 56, 56, 56, 17, 56, 56, 56, + 56, 17, 17, 17, 56, 56, 56, 56, 17, 17, + 17, 56, 56, 17, 17, 17, 56, 17, 17, 56, + 17, 56, 17, 56, 56, 0, 56 } ; -static yyconst flex_int16_t yy_nxt[89] = +static yyconst flex_int16_t yy_nxt[127] = { 0, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 12, 14, 15, 16, 17, 12, 12, 12, 18, 12, 12, 12, 12, 12, 12, 12, 12, 19, 12, 20, - 24, 46, 45, 44, 43, 42, 41, 40, 39, 38, - 37, 36, 35, 34, 33, 32, 31, 30, 29, 28, - 27, 26, 25, 23, 22, 21, 47, 3, 47, 47, - 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, - 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, - 47, 47, 47, 47, 47, 47, 47, 47 + 4, 21, 22, 23, 28, 29, 30, 32, 27, 27, + 34, 33, 39, 27, 41, 28, 29, 30, 27, 40, + 27, 27, 44, 27, 46, 49, 45, 53, 51, 27, + 27, 27, 27, 27, 55, 27, 54, 52, 27, 50, + 27, 48, 47, 27, 43, 42, 27, 37, 36, 27, + 27, 38, 37, 36, 27, 35, 31, 26, 25, 24, + 56, 3, 56, 56, 56, 56, 56, 56, 56, 56, + + 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, + 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, + 56, 56, 56, 56, 56, 56 } ; -static yyconst flex_int16_t yy_chk[89] = +static yyconst flex_int16_t yy_chk[127] = { 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 48, 45, 43, 42, 41, 39, 38, 36, 35, 34, - 33, 32, 31, 30, 28, 27, 26, 22, 20, 19, - 18, 17, 15, 9, 8, 6, 3, 47, 47, 47, - 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, - 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, - 47, 47, 47, 47, 47, 47, 47, 47 + 1, 1, 1, 1, 12, 12, 12, 17, 19, 34, + 19, 18, 32, 57, 34, 17, 17, 17, 18, 33, + 32, 33, 39, 40, 41, 45, 40, 51, 48, 39, + 41, 45, 48, 51, 54, 53, 52, 50, 49, 47, + 46, 44, 43, 42, 38, 37, 36, 30, 29, 28, + 27, 25, 23, 22, 21, 20, 15, 9, 8, 6, + 3, 56, 56, 56, 56, 56, 56, 56, 56, 56, + + 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, + 56, 56, 56, 56, 56, 56, 56, 56, 56, 56, + 56, 56, 56, 56, 56, 56 } ; static yy_state_type yy_last_accepting_state; @@ -488,7 +502,7 @@ char *yytext; #include #include #include "slmod_sparql_parser.h" -#line 492 "spotlight_rawquery_lexer.c" +#line 506 "spotlight_rawquery_lexer.c" #define INITIAL 0 @@ -670,9 +684,9 @@ YY_DECL register char *yy_cp, *yy_bp; register int yy_act; -#line 15 "spotlight_rawquery_lexer.l" +#line 24 "spotlight_rawquery_lexer.l" -#line 676 "spotlight_rawquery_lexer.c" +#line 690 "spotlight_rawquery_lexer.c" if ( !(yy_init) ) { @@ -725,13 +739,13 @@ yy_match: while ( yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state ) { yy_current_state = (int) yy_def[yy_current_state]; - if ( yy_current_state >= 48 ) + if ( yy_current_state >= 57 ) yy_c = yy_meta[(unsigned int) yy_c]; } yy_current_state = yy_nxt[yy_base[yy_current_state] + (unsigned int) yy_c]; ++yy_cp; } - while ( yy_base[yy_current_state] != 58 ); + while ( yy_base[yy_current_state] != 92 ); yy_find_action: yy_act = yy_accept[yy_current_state]; @@ -757,91 +771,91 @@ do_action: /* This label is used only to access EOF actions. */ case 1: YY_RULE_SETUP -#line 16 "spotlight_rawquery_lexer.l" -return FUNC_INRANGE; +#line 25 "spotlight_rawquery_lexer.l" +return FUNC_INRANGE; YY_BREAK case 2: YY_RULE_SETUP -#line 17 "spotlight_rawquery_lexer.l" +#line 26 "spotlight_rawquery_lexer.l" return DATE_ISO; YY_BREAK case 3: YY_RULE_SETUP -#line 18 "spotlight_rawquery_lexer.l" +#line 27 "spotlight_rawquery_lexer.l" {yylval.bval = false; return BOOL;} YY_BREAK case 4: YY_RULE_SETUP -#line 19 "spotlight_rawquery_lexer.l" +#line 28 "spotlight_rawquery_lexer.l" {yylval.bval = true; return BOOL;} YY_BREAK case 5: YY_RULE_SETUP -#line 20 "spotlight_rawquery_lexer.l" -{yylval.sval = talloc_strdup(ssp_slq, yytext); return WORD;} +#line 29 "spotlight_rawquery_lexer.l" +return QUOTE; YY_BREAK case 6: YY_RULE_SETUP -#line 21 "spotlight_rawquery_lexer.l" -return QUOTE; +#line 30 "spotlight_rawquery_lexer.l" +return OBRACE; YY_BREAK case 7: YY_RULE_SETUP -#line 22 "spotlight_rawquery_lexer.l" -return OBRACE; +#line 31 "spotlight_rawquery_lexer.l" +return CBRACE; YY_BREAK case 8: YY_RULE_SETUP -#line 23 "spotlight_rawquery_lexer.l" -return CBRACE; +#line 32 "spotlight_rawquery_lexer.l" +return AND; YY_BREAK case 9: YY_RULE_SETUP -#line 24 "spotlight_rawquery_lexer.l" -return AND; +#line 33 "spotlight_rawquery_lexer.l" +return OR; YY_BREAK case 10: YY_RULE_SETUP -#line 25 "spotlight_rawquery_lexer.l" -return OR; +#line 34 "spotlight_rawquery_lexer.l" +return EQUAL; YY_BREAK case 11: YY_RULE_SETUP -#line 26 "spotlight_rawquery_lexer.l" -return EQUAL; +#line 35 "spotlight_rawquery_lexer.l" +return UNEQUAL; YY_BREAK case 12: YY_RULE_SETUP -#line 27 "spotlight_rawquery_lexer.l" -return UNEQUAL; +#line 36 "spotlight_rawquery_lexer.l" +return LT; YY_BREAK case 13: YY_RULE_SETUP -#line 28 "spotlight_rawquery_lexer.l" -return LT; +#line 37 "spotlight_rawquery_lexer.l" +return GT; YY_BREAK case 14: YY_RULE_SETUP -#line 29 "spotlight_rawquery_lexer.l" -return GT; +#line 38 "spotlight_rawquery_lexer.l" +return COMMA; YY_BREAK case 15: YY_RULE_SETUP -#line 30 "spotlight_rawquery_lexer.l" -return COMMA; +#line 39 "spotlight_rawquery_lexer.l" +{yylval.sval = talloc_strdup(ssp_slq, yytext); return WORD;} YY_BREAK case 16: /* rule 16 can match eol */ YY_RULE_SETUP -#line 31 "spotlight_rawquery_lexer.l" +#line 40 "spotlight_rawquery_lexer.l" /* ignore */ YY_BREAK case 17: YY_RULE_SETUP -#line 32 "spotlight_rawquery_lexer.l" +#line 41 "spotlight_rawquery_lexer.l" ECHO; YY_BREAK -#line 845 "spotlight_rawquery_lexer.c" +#line 859 "spotlight_rawquery_lexer.c" case YY_STATE_EOF(INITIAL): yyterminate(); @@ -1133,7 +1147,7 @@ static int yy_get_next_buffer (void) while ( yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state ) { yy_current_state = (int) yy_def[yy_current_state]; - if ( yy_current_state >= 48 ) + if ( yy_current_state >= 57 ) yy_c = yy_meta[(unsigned int) yy_c]; } yy_current_state = yy_nxt[yy_base[yy_current_state] + (unsigned int) yy_c]; @@ -1161,11 +1175,11 @@ static int yy_get_next_buffer (void) while ( yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state ) { yy_current_state = (int) yy_def[yy_current_state]; - if ( yy_current_state >= 48 ) + if ( yy_current_state >= 57 ) yy_c = yy_meta[(unsigned int) yy_c]; } yy_current_state = yy_nxt[yy_base[yy_current_state] + (unsigned int) yy_c]; - yy_is_jam = (yy_current_state == 47); + yy_is_jam = (yy_current_state == 56); return yy_is_jam ? 0 : yy_current_state; } @@ -1839,7 +1853,7 @@ void yyfree (void * ptr ) #define YYTABLES_NAME "yytables" -#line 32 "spotlight_rawquery_lexer.l" +#line 41 "spotlight_rawquery_lexer.l" diff --git a/etc/spotlight/spotlight_rawquery_lexer.l b/etc/spotlight/spotlight_rawquery_lexer.l index bbfcad50..80ce17fd 100644 --- a/etc/spotlight/spotlight_rawquery_lexer.l +++ b/etc/spotlight/spotlight_rawquery_lexer.l @@ -12,12 +12,20 @@ #include "slmod_sparql_parser.h" %} +ASC [a-zA-Z0-9_\*\:\-\.] +U [\x80-\xbf] +U2 [\xc2-\xdf] +U3 [\xe0-\xef] +U4 [\xf0-\xf4] + +UANY {ASC}|{U2}{U}|{U3}{U}{U}|{U4}{U}{U}{U} +UONLY {U2}{U}|{U3}{U}{U}|{U4}{U}{U}{U} + %% InRange return FUNC_INRANGE; \$time\.iso return DATE_ISO; false {yylval.bval = false; return BOOL;} true {yylval.bval = true; return BOOL;} -[a-zA-Z0-9_\*\:\-\.]+ {yylval.sval = talloc_strdup(ssp_slq, yytext); return WORD;} \" return QUOTE; \( return OBRACE; \) return CBRACE; @@ -28,5 +36,6 @@ true {yylval.bval = true; return BOOL;} \< return LT; \> return GT; \, return COMMA; +{UANY}+ {yylval.sval = talloc_strdup(ssp_slq, yytext); return WORD;} [ \t\n] /* ignore */ %%