Add UTF8 support

diff --git a/etc/spotlight/spotlight_rawquery_lexer.c b/etc/spotlight/spotlight_rawquery_lexer.c
index c278fba654705d5b112db7dca62c90ae81237e73..350df439369003486eb71dfb542fe94648cb1684 100644 (file)
--- a/etc/spotlight/spotlight_rawquery_lexer.c
+++ b/etc/spotlight/spotlight_rawquery_lexer.c
@@ -374,13 +374,14 @@ struct yy_trans_info
        flex_int32_t yy_verify;
        flex_int32_t yy_nxt;
        };
-static yyconst flex_int16_t yy_accept[48] =
+static yyconst flex_int16_t yy_accept[57] =
     {   0,
-        0,    0,   18,   17,   16,   17,    6,   17,   17,    7,
-        8,    5,   15,   13,   17,   14,    5,    5,    5,   17,
-       12,    0,    9,    5,   11,    5,    5,    5,   10,    0,
-        5,    5,    5,    0,    5,    5,    4,    0,    5,    3,
-        0,    5,    0,    1,    0,    2,    0
+        0,    0,   18,   17,   16,   17,    5,   17,   17,    6,
+        7,   15,   14,   12,   17,   13,   15,   15,   15,   17,
+       17,   17,   17,   11,    0,    8,   15,    0,    0,    0,
+       10,   15,   15,   15,    9,    0,    0,    0,   15,   15,
+       15,    0,    0,   15,   15,    4,    0,   15,    3,    0,
+       15,    0,    1,    0,    2,    0
     } ;
 
 static yyconst flex_int32_t yy_ec[256] =
@@ -398,72 +399,85 @@ static yyconst flex_int32_t yy_ec[256] =
 
        18,   19,   20,    9,   21,    9,    9,   22,   23,   24,
        25,    9,    9,   26,   27,   28,   29,    9,    9,    9,
-        9,    9,    1,   30,    1,    1,    1,    1,    1,    1,
-        1,    1,    1,    1,    1,    1,    1,    1,    1,    1,
-        1,    1,    1,    1,    1,    1,    1,    1,    1,    1,
-        1,    1,    1,    1,    1,    1,    1,    1,    1,    1,
-        1,    1,    1,    1,    1,    1,    1,    1,    1,    1,
-        1,    1,    1,    1,    1,    1,    1,    1,    1,    1,
-        1,    1,    1,    1,    1,    1,    1,    1,    1,    1,
-        1,    1,    1,    1,    1,    1,    1,    1,    1,    1,
-
-        1,    1,    1,    1,    1,    1,    1,    1,    1,    1,
-        1,    1,    1,    1,    1,    1,    1,    1,    1,    1,
-        1,    1,    1,    1,    1,    1,    1,    1,    1,    1,
-        1,    1,    1,    1,    1,    1,    1,    1,    1,    1,
-        1,    1,    1,    1,    1,    1,    1,    1,    1,    1,
+        9,    9,    1,   30,    1,    1,    1,   31,   31,   31,
+       31,   31,   31,   31,   31,   31,   31,   31,   31,   31,
+       31,   31,   31,   31,   31,   31,   31,   31,   31,   31,
+       31,   31,   31,   31,   31,   31,   31,   31,   31,   31,
+       31,   31,   31,   31,   31,   31,   31,   31,   31,   31,
+       31,   31,   31,   31,   31,   31,   31,   31,   31,   31,
+       31,   31,   31,   31,   31,   31,   31,   31,   31,   31,
+       31,    1,    1,   32,   32,   32,   32,   32,   32,   32,
+
+       32,   32,   32,   32,   32,   32,   32,   32,   32,   32,
+       32,   32,   32,   32,   32,   32,   32,   32,   32,   32,
+       32,   32,   32,   33,   33,   33,   33,   33,   33,   33,
+       33,   33,   33,   33,   33,   33,   33,   33,   33,   34,
+       34,   34,   34,   34,    1,    1,    1,    1,    1,    1,
         1,    1,    1,    1,    1
     } ;
 
-static yyconst flex_int32_t yy_meta[31] =
+static yyconst flex_int32_t yy_meta[35] =
     {   0,
         1,    1,    1,    1,    1,    1,    1,    1,    2,    1,
         2,    1,    1,    1,    2,    2,    2,    2,    2,    2,
-        2,    2,    2,    2,    2,    2,    2,    2,    2,    1
+        2,    2,    2,    2,    2,    2,    2,    2,    2,    1,
+        1,    2,    2,    2
     } ;
 
-static yyconst flex_int16_t yy_base[49] =
+static yyconst flex_int16_t yy_base[58] =
     {   0,
-        0,    0,   57,   58,   58,   43,   58,   27,   48,   58,
-       58,    0,   58,   58,   40,   58,   28,   34,   24,   19,
-       58,   27,   58,    0,   58,   31,   24,   16,   58,   21,
-       26,   15,   23,   22,   15,   20,    0,   26,   16,    0,
-       14,   16,    6,    0,    7,   58,   58,   29
+        0,    0,   91,   92,   92,   77,   92,   61,   82,   92,
+       92,    3,   92,   92,   74,   92,   14,   25,   15,   56,
+       54,   53,   52,   92,   61,   92,   57,   49,   48,   47,
+       92,   27,   28,   16,   92,   46,   45,   52,   36,   30,
+       37,   43,   55,   48,   38,   47,   59,   39,   45,   47,
+       40,   40,   42,   40,   92,   92,   42
     } ;
 
-static yyconst flex_int16_t yy_def[49] =
+static yyconst flex_int16_t yy_def[58] =
     {   0,
-       47,    1,   47,   47,   47,   47,   47,   47,   47,   47,
-       47,   48,   47,   47,   47,   47,   48,   48,   48,   47,
-       47,   47,   47,   48,   47,   48,   48,   48,   47,   47,
-       48,   48,   48,   47,   48,   48,   48,   47,   48,   48,
-       47,   48,   47,   48,   47,   47,    0,   47
+       56,    1,   56,   56,   56,   56,   56,   56,   56,   56,
+       56,   57,   56,   56,   56,   56,   57,   17,   17,   56,
+       56,   56,   56,   56,   56,   56,   17,   56,   56,   56,
+       56,   17,   17,   17,   56,   56,   56,   56,   17,   17,
+       17,   56,   56,   17,   17,   17,   56,   17,   17,   56,
+       17,   56,   17,   56,   56,    0,   56
     } ;
 
-static yyconst flex_int16_t yy_nxt[89] =
+static yyconst flex_int16_t yy_nxt[127] =
     {   0,
         4,    5,    6,    7,    8,    9,   10,   11,   12,   13,
        12,   14,   15,   16,   17,   12,   12,   12,   18,   12,
        12,   12,   12,   12,   12,   12,   12,   19,   12,   20,
-       24,   46,   45,   44,   43,   42,   41,   40,   39,   38,
-       37,   36,   35,   34,   33,   32,   31,   30,   29,   28,
-       27,   26,   25,   23,   22,   21,   47,    3,   47,   47,
-       47,   47,   47,   47,   47,   47,   47,   47,   47,   47,
-       47,   47,   47,   47,   47,   47,   47,   47,   47,   47,
-       47,   47,   47,   47,   47,   47,   47,   47
+        4,   21,   22,   23,   28,   29,   30,   32,   27,   27,
+       34,   33,   39,   27,   41,   28,   29,   30,   27,   40,
+       27,   27,   44,   27,   46,   49,   45,   53,   51,   27,
+       27,   27,   27,   27,   55,   27,   54,   52,   27,   50,
+       27,   48,   47,   27,   43,   42,   27,   37,   36,   27,
+       27,   38,   37,   36,   27,   35,   31,   26,   25,   24,
+       56,    3,   56,   56,   56,   56,   56,   56,   56,   56,
+
+       56,   56,   56,   56,   56,   56,   56,   56,   56,   56,
+       56,   56,   56,   56,   56,   56,   56,   56,   56,   56,
+       56,   56,   56,   56,   56,   56
     } ;
 
-static yyconst flex_int16_t yy_chk[89] =
+static yyconst flex_int16_t yy_chk[127] =
     {   0,
         1,    1,    1,    1,    1,    1,    1,    1,    1,    1,
         1,    1,    1,    1,    1,    1,    1,    1,    1,    1,
         1,    1,    1,    1,    1,    1,    1,    1,    1,    1,
-       48,   45,   43,   42,   41,   39,   38,   36,   35,   34,
-       33,   32,   31,   30,   28,   27,   26,   22,   20,   19,
-       18,   17,   15,    9,    8,    6,    3,   47,   47,   47,
-       47,   47,   47,   47,   47,   47,   47,   47,   47,   47,
-       47,   47,   47,   47,   47,   47,   47,   47,   47,   47,
-       47,   47,   47,   47,   47,   47,   47,   47
+        1,    1,    1,    1,   12,   12,   12,   17,   19,   34,
+       19,   18,   32,   57,   34,   17,   17,   17,   18,   33,
+       32,   33,   39,   40,   41,   45,   40,   51,   48,   39,
+       41,   45,   48,   51,   54,   53,   52,   50,   49,   47,
+       46,   44,   43,   42,   38,   37,   36,   30,   29,   28,
+       27,   25,   23,   22,   21,   20,   15,    9,    8,    6,
+        3,   56,   56,   56,   56,   56,   56,   56,   56,   56,
+
+       56,   56,   56,   56,   56,   56,   56,   56,   56,   56,
+       56,   56,   56,   56,   56,   56,   56,   56,   56,   56,
+       56,   56,   56,   56,   56,   56
     } ;
 
 static yy_state_type yy_last_accepting_state;
@@ -488,7 +502,7 @@ char *yytext;
 #include <atalk/talloc.h>
 #include <atalk/spotlight.h>
 #include "slmod_sparql_parser.h"
-#line 492 "spotlight_rawquery_lexer.c"
+#line 506 "spotlight_rawquery_lexer.c"
 
 #define INITIAL 0
 
@@ -670,9 +684,9 @@ YY_DECL
        register char *yy_cp, *yy_bp;
        register int yy_act;
     
-#line 15 "spotlight_rawquery_lexer.l"
+#line 24 "spotlight_rawquery_lexer.l"
 
-#line 676 "spotlight_rawquery_lexer.c"
+#line 690 "spotlight_rawquery_lexer.c"
 
        if ( !(yy_init) )
                {
@@ -725,13 +739,13 @@ yy_match:
                        while ( yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state )
                                {
                                yy_current_state = (int) yy_def[yy_current_state];
-                               if ( yy_current_state >= 48 )
+                               if ( yy_current_state >= 57 )
                                        yy_c = yy_meta[(unsigned int) yy_c];
                                }
                        yy_current_state = yy_nxt[yy_base[yy_current_state] + (unsigned int) yy_c];
                        ++yy_cp;
                        }
-               while ( yy_base[yy_current_state] != 58 );
+               while ( yy_base[yy_current_state] != 92 );
 
 yy_find_action:
                yy_act = yy_accept[yy_current_state];
@@ -757,91 +771,91 @@ do_action:        /* This label is used only to access EOF actions. */
 
 case 1:
 YY_RULE_SETUP
-#line 16 "spotlight_rawquery_lexer.l"
-return FUNC_INRANGE;
+#line 25 "spotlight_rawquery_lexer.l"
+return FUNC_INRANGE;                                       
        YY_BREAK
 case 2:
 YY_RULE_SETUP
-#line 17 "spotlight_rawquery_lexer.l"
+#line 26 "spotlight_rawquery_lexer.l"
 return DATE_ISO;
        YY_BREAK
 case 3:
 YY_RULE_SETUP
-#line 18 "spotlight_rawquery_lexer.l"
+#line 27 "spotlight_rawquery_lexer.l"
 {yylval.bval = false; return BOOL;}
        YY_BREAK
 case 4:
 YY_RULE_SETUP
-#line 19 "spotlight_rawquery_lexer.l"
+#line 28 "spotlight_rawquery_lexer.l"
 {yylval.bval = true; return BOOL;}
        YY_BREAK
 case 5:
 YY_RULE_SETUP
-#line 20 "spotlight_rawquery_lexer.l"
-{yylval.sval = talloc_strdup(ssp_slq, yytext); return WORD;}
+#line 29 "spotlight_rawquery_lexer.l"
+return QUOTE;
        YY_BREAK
 case 6:
 YY_RULE_SETUP
-#line 21 "spotlight_rawquery_lexer.l"
-return QUOTE;
+#line 30 "spotlight_rawquery_lexer.l"
+return OBRACE;
        YY_BREAK
 case 7:
 YY_RULE_SETUP
-#line 22 "spotlight_rawquery_lexer.l"
-return OBRACE;
+#line 31 "spotlight_rawquery_lexer.l"
+return CBRACE;
        YY_BREAK
 case 8:
 YY_RULE_SETUP
-#line 23 "spotlight_rawquery_lexer.l"
-return CBRACE;
+#line 32 "spotlight_rawquery_lexer.l"
+return AND;
        YY_BREAK
 case 9:
 YY_RULE_SETUP
-#line 24 "spotlight_rawquery_lexer.l"
-return AND;
+#line 33 "spotlight_rawquery_lexer.l"
+return OR;
        YY_BREAK
 case 10:
 YY_RULE_SETUP
-#line 25 "spotlight_rawquery_lexer.l"
-return OR;
+#line 34 "spotlight_rawquery_lexer.l"
+return EQUAL;
        YY_BREAK
 case 11:
 YY_RULE_SETUP
-#line 26 "spotlight_rawquery_lexer.l"
-return EQUAL;
+#line 35 "spotlight_rawquery_lexer.l"
+return UNEQUAL;
        YY_BREAK
 case 12:
 YY_RULE_SETUP
-#line 27 "spotlight_rawquery_lexer.l"
-return UNEQUAL;
+#line 36 "spotlight_rawquery_lexer.l"
+return LT;
        YY_BREAK
 case 13:
 YY_RULE_SETUP
-#line 28 "spotlight_rawquery_lexer.l"
-return LT;
+#line 37 "spotlight_rawquery_lexer.l"
+return GT;
        YY_BREAK
 case 14:
 YY_RULE_SETUP
-#line 29 "spotlight_rawquery_lexer.l"
-return GT;
+#line 38 "spotlight_rawquery_lexer.l"
+return COMMA;
        YY_BREAK
 case 15:
 YY_RULE_SETUP
-#line 30 "spotlight_rawquery_lexer.l"
-return COMMA;
+#line 39 "spotlight_rawquery_lexer.l"
+{yylval.sval = talloc_strdup(ssp_slq, yytext); return WORD;}
        YY_BREAK
 case 16:
 /* rule 16 can match eol */
 YY_RULE_SETUP
-#line 31 "spotlight_rawquery_lexer.l"
+#line 40 "spotlight_rawquery_lexer.l"
 /* ignore */
        YY_BREAK
 case 17:
 YY_RULE_SETUP
-#line 32 "spotlight_rawquery_lexer.l"
+#line 41 "spotlight_rawquery_lexer.l"
 ECHO;
        YY_BREAK
-#line 845 "spotlight_rawquery_lexer.c"
+#line 859 "spotlight_rawquery_lexer.c"
 case YY_STATE_EOF(INITIAL):
        yyterminate();
 
@@ -1133,7 +1147,7 @@ static int yy_get_next_buffer (void)
                while ( yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state )
                        {
                        yy_current_state = (int) yy_def[yy_current_state];
-                       if ( yy_current_state >= 48 )
+                       if ( yy_current_state >= 57 )
                                yy_c = yy_meta[(unsigned int) yy_c];
                        }
                yy_current_state = yy_nxt[yy_base[yy_current_state] + (unsigned int) yy_c];
@@ -1161,11 +1175,11 @@ static int yy_get_next_buffer (void)
        while ( yy_chk[yy_base[yy_current_state] + yy_c] != yy_current_state )
                {
                yy_current_state = (int) yy_def[yy_current_state];
-               if ( yy_current_state >= 48 )
+               if ( yy_current_state >= 57 )
                        yy_c = yy_meta[(unsigned int) yy_c];
                }
        yy_current_state = yy_nxt[yy_base[yy_current_state] + (unsigned int) yy_c];
-       yy_is_jam = (yy_current_state == 47);
+       yy_is_jam = (yy_current_state == 56);
 
        return yy_is_jam ? 0 : yy_current_state;
 }
@@ -1839,7 +1853,7 @@ void yyfree (void * ptr )
 
 #define YYTABLES_NAME "yytables"
 
-#line 32 "spotlight_rawquery_lexer.l"
+#line 41 "spotlight_rawquery_lexer.l"
 
 
 

diff --git a/etc/spotlight/spotlight_rawquery_lexer.l b/etc/spotlight/spotlight_rawquery_lexer.l
index bbfcad50a963b6a8013d06c541abecb6f3d54474..80ce17fd7a3699455800a3c1bbec8ed0fe9ab967 100644 (file)
--- a/etc/spotlight/spotlight_rawquery_lexer.l
+++ b/etc/spotlight/spotlight_rawquery_lexer.l
@@ -12,12 +12,20 @@
 #include "slmod_sparql_parser.h"
 %}
 
+ASC     [a-zA-Z0-9_\*\:\-\.]
+U       [\x80-\xbf]
+U2      [\xc2-\xdf]
+U3      [\xe0-\xef]
+U4      [\xf0-\xf4]
+
+UANY    {ASC}|{U2}{U}|{U3}{U}{U}|{U4}{U}{U}{U}
+UONLY   {U2}{U}|{U3}{U}{U}|{U4}{U}{U}{U}
+
 %%
 InRange                               return FUNC_INRANGE;
 \$time\.iso                           return DATE_ISO;
 false                                 {yylval.bval = false; return BOOL;}
 true                                  {yylval.bval = true; return BOOL;}
-[a-zA-Z0-9_\*\:\-\.]+                 {yylval.sval = talloc_strdup(ssp_slq, yytext); return WORD;}
 \"                                    return QUOTE;
 \(                                    return OBRACE;
 \)                                    return CBRACE;
@@ -28,5 +36,6 @@ true                                  {yylval.bval = true; return BOOL;}
 \<                                    return LT;
 \>                                    return GT;
 \,                                    return COMMA;
+{UANY}+                               {yylval.sval = talloc_strdup(ssp_slq, yytext); return WORD;}
 [ \t\n]                               /* ignore */
 %%