diff options
Diffstat (limited to 'venv/lib/python3.11/site-packages/pygments/lexers/sql.py')
| -rw-r--r-- | venv/lib/python3.11/site-packages/pygments/lexers/sql.py | 1027 |
1 files changed, 1027 insertions, 0 deletions
diff --git a/venv/lib/python3.11/site-packages/pygments/lexers/sql.py b/venv/lib/python3.11/site-packages/pygments/lexers/sql.py new file mode 100644 index 0000000..2880841 --- /dev/null +++ b/venv/lib/python3.11/site-packages/pygments/lexers/sql.py @@ -0,0 +1,1027 @@ +""" + pygments.lexers.sql + ~~~~~~~~~~~~~~~~~~~ + + Lexers for various SQL dialects and related interactive sessions. + + Postgres specific lexers: + + `PostgresLexer` + A SQL lexer for the PostgreSQL dialect. Differences w.r.t. the SQL + lexer are: + + - keywords and data types list parsed from the PG docs (run the + `_postgres_builtins` module to update them); + - Content of $-strings parsed using a specific lexer, e.g. the content + of a PL/Python function is parsed using the Python lexer; + - parse PG specific constructs: E-strings, $-strings, U&-strings, + different operators and punctuation. + + `PlPgsqlLexer` + A lexer for the PL/pgSQL language. Adds a few specific construct on + top of the PG SQL lexer (such as <<label>>). + + `PostgresConsoleLexer` + A lexer to highlight an interactive psql session: + + - identifies the prompt and does its best to detect the end of command + in multiline statement where not all the lines are prefixed by a + prompt, telling them apart from the output; + - highlights errors in the output and notification levels; + - handles psql backslash commands. + + `PostgresExplainLexer` + A lexer to highlight Postgres execution plan. + + The ``tests/examplefiles`` contains a few test files with data to be + parsed by these lexers. + + :copyright: Copyright 2006-2023 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +import re + +from pygments.lexer import Lexer, RegexLexer, do_insertions, bygroups, words +from pygments.token import Punctuation, Whitespace, Text, Comment, Operator, \ + Keyword, Name, String, Number, Generic, Literal +from pygments.lexers import get_lexer_by_name, ClassNotFound + +from pygments.lexers._postgres_builtins import KEYWORDS, DATATYPES, \ + PSEUDO_TYPES, PLPGSQL_KEYWORDS, EXPLAIN_KEYWORDS +from pygments.lexers._mysql_builtins import \ + MYSQL_CONSTANTS, \ + MYSQL_DATATYPES, \ + MYSQL_FUNCTIONS, \ + MYSQL_KEYWORDS, \ + MYSQL_OPTIMIZER_HINTS + +from pygments.lexers import _tsql_builtins + + +__all__ = ['PostgresLexer', 'PlPgsqlLexer', 'PostgresConsoleLexer', + 'PostgresExplainLexer', 'SqlLexer', 'TransactSqlLexer', + 'MySqlLexer', 'SqliteConsoleLexer', 'RqlLexer'] + +line_re = re.compile('.*?\n') +sqlite_prompt_re = re.compile(r'^(?:sqlite| ...)>(?= )') + +language_re = re.compile(r"\s+LANGUAGE\s+'?(\w+)'?", re.IGNORECASE) + +do_re = re.compile(r'\bDO\b', re.IGNORECASE) + +# Regular expressions for analyse_text() +name_between_bracket_re = re.compile(r'\[[a-zA-Z_]\w*\]') +name_between_backtick_re = re.compile(r'`[a-zA-Z_]\w*`') +tsql_go_re = re.compile(r'\bgo\b', re.IGNORECASE) +tsql_declare_re = re.compile(r'\bdeclare\s+@', re.IGNORECASE) +tsql_variable_re = re.compile(r'@[a-zA-Z_]\w*\b') + + +def language_callback(lexer, match): + """Parse the content of a $-string using a lexer + + The lexer is chosen looking for a nearby LANGUAGE or assumed as + plpgsql if inside a DO statement and no LANGUAGE has been found. + """ + lx = None + m = language_re.match(lexer.text[match.end():match.end()+100]) + if m is not None: + lx = lexer._get_lexer(m.group(1)) + else: + m = list(language_re.finditer( + lexer.text[max(0, match.start()-100):match.start()])) + if m: + lx = lexer._get_lexer(m[-1].group(1)) + else: + m = list(do_re.finditer( + lexer.text[max(0, match.start()-25):match.start()])) + if m: + lx = lexer._get_lexer('plpgsql') + + # 1 = $, 2 = delimiter, 3 = $ + yield (match.start(1), String, match.group(1)) + yield (match.start(2), String.Delimiter, match.group(2)) + yield (match.start(3), String, match.group(3)) + # 4 = string contents + if lx: + yield from lx.get_tokens_unprocessed(match.group(4)) + else: + yield (match.start(4), String, match.group(4)) + # 5 = $, 6 = delimiter, 7 = $ + yield (match.start(5), String, match.group(5)) + yield (match.start(6), String.Delimiter, match.group(6)) + yield (match.start(7), String, match.group(7)) + + +class PostgresBase: + """Base class for Postgres-related lexers. + + This is implemented as a mixin to avoid the Lexer metaclass kicking in. + this way the different lexer don't have a common Lexer ancestor. If they + had, _tokens could be created on this ancestor and not updated for the + other classes, resulting e.g. in PL/pgSQL parsed as SQL. This shortcoming + seem to suggest that regexp lexers are not really subclassable. + """ + def get_tokens_unprocessed(self, text, *args): + # Have a copy of the entire text to be used by `language_callback`. + self.text = text + yield from super().get_tokens_unprocessed(text, *args) + + def _get_lexer(self, lang): + if lang.lower() == 'sql': + return get_lexer_by_name('postgresql', **self.options) + + tries = [lang] + if lang.startswith('pl'): + tries.append(lang[2:]) + if lang.endswith('u'): + tries.append(lang[:-1]) + if lang.startswith('pl') and lang.endswith('u'): + tries.append(lang[2:-1]) + + for lx in tries: + try: + return get_lexer_by_name(lx, **self.options) + except ClassNotFound: + pass + else: + # TODO: better logging + # print >>sys.stderr, "language not found:", lang + return None + + +class PostgresLexer(PostgresBase, RegexLexer): + """ + Lexer for the PostgreSQL dialect of SQL. + + .. versionadded:: 1.5 + """ + + name = 'PostgreSQL SQL dialect' + aliases = ['postgresql', 'postgres'] + mimetypes = ['text/x-postgresql'] + + flags = re.IGNORECASE + tokens = { + 'root': [ + (r'\s+', Whitespace), + (r'--.*\n?', Comment.Single), + (r'/\*', Comment.Multiline, 'multiline-comments'), + (r'(' + '|'.join(s.replace(" ", r"\s+") + for s in DATATYPES + PSEUDO_TYPES) + r')\b', + Name.Builtin), + (words(KEYWORDS, suffix=r'\b'), Keyword), + (r'[+*/<>=~!@#%^&|`?-]+', Operator), + (r'::', Operator), # cast + (r'\$\d+', Name.Variable), + (r'([0-9]*\.[0-9]*|[0-9]+)(e[+-]?[0-9]+)?', Number.Float), + (r'[0-9]+', Number.Integer), + (r"((?:E|U&)?)(')", bygroups(String.Affix, String.Single), 'string'), + # quoted identifier + (r'((?:U&)?)(")', bygroups(String.Affix, String.Name), 'quoted-ident'), + (r'(?s)(\$)([^$]*)(\$)(.*?)(\$)(\2)(\$)', language_callback), + (r'[a-z_]\w*', Name), + + # psql variable in SQL + (r""":(['"]?)[a-z]\w*\b\1""", Name.Variable), + + (r'[;:()\[\]{},.]', Punctuation), + ], + 'multiline-comments': [ + (r'/\*', Comment.Multiline, 'multiline-comments'), + (r'\*/', Comment.Multiline, '#pop'), + (r'[^/*]+', Comment.Multiline), + (r'[/*]', Comment.Multiline) + ], + 'string': [ + (r"[^']+", String.Single), + (r"''", String.Single), + (r"'", String.Single, '#pop'), + ], + 'quoted-ident': [ + (r'[^"]+', String.Name), + (r'""', String.Name), + (r'"', String.Name, '#pop'), + ], + } + + +class PlPgsqlLexer(PostgresBase, RegexLexer): + """ + Handle the extra syntax in Pl/pgSQL language. + + .. versionadded:: 1.5 + """ + name = 'PL/pgSQL' + aliases = ['plpgsql'] + mimetypes = ['text/x-plpgsql'] + + flags = re.IGNORECASE + tokens = {k: l[:] for (k, l) in PostgresLexer.tokens.items()} + + # extend the keywords list + for i, pattern in enumerate(tokens['root']): + if pattern[1] == Keyword: + tokens['root'][i] = ( + words(KEYWORDS + PLPGSQL_KEYWORDS, suffix=r'\b'), + Keyword) + del i + break + else: + assert 0, "SQL keywords not found" + + # Add specific PL/pgSQL rules (before the SQL ones) + tokens['root'][:0] = [ + (r'\%[a-z]\w*\b', Name.Builtin), # actually, a datatype + (r':=', Operator), + (r'\<\<[a-z]\w*\>\>', Name.Label), + (r'\#[a-z]\w*\b', Keyword.Pseudo), # #variable_conflict + ] + + +class PsqlRegexLexer(PostgresBase, RegexLexer): + """ + Extend the PostgresLexer adding support specific for psql commands. + + This is not a complete psql lexer yet as it lacks prompt support + and output rendering. + """ + + name = 'PostgreSQL console - regexp based lexer' + aliases = [] # not public + + flags = re.IGNORECASE + tokens = {k: l[:] for (k, l) in PostgresLexer.tokens.items()} + + tokens['root'].append( + (r'\\[^\s]+', Keyword.Pseudo, 'psql-command')) + tokens['psql-command'] = [ + (r'\n', Text, 'root'), + (r'\s+', Whitespace), + (r'\\[^\s]+', Keyword.Pseudo), + (r""":(['"]?)[a-z]\w*\b\1""", Name.Variable), + (r"'(''|[^'])*'", String.Single), + (r"`([^`])*`", String.Backtick), + (r"[^\s]+", String.Symbol), + ] + + +re_prompt = re.compile(r'^(\S.*?)??[=\-\(\$\'\"][#>]') +re_psql_command = re.compile(r'\s*\\') +re_end_command = re.compile(r';\s*(--.*?)?$') +re_psql_command = re.compile(r'(\s*)(\\.+?)(\s+)$') +re_error = re.compile(r'(ERROR|FATAL):') +re_message = re.compile( + r'((?:DEBUG|INFO|NOTICE|WARNING|ERROR|' + r'FATAL|HINT|DETAIL|CONTEXT|LINE [0-9]+):)(.*?\n)') + + +class lookahead: + """Wrap an iterator and allow pushing back an item.""" + def __init__(self, x): + self.iter = iter(x) + self._nextitem = None + + def __iter__(self): + return self + + def send(self, i): + self._nextitem = i + return i + + def __next__(self): + if self._nextitem is not None: + ni = self._nextitem + self._nextitem = None + return ni + return next(self.iter) + next = __next__ + + +class PostgresConsoleLexer(Lexer): + """ + Lexer for psql sessions. + + .. versionadded:: 1.5 + """ + + name = 'PostgreSQL console (psql)' + aliases = ['psql', 'postgresql-console', 'postgres-console'] + mimetypes = ['text/x-postgresql-psql'] + + def get_tokens_unprocessed(self, data): + sql = PsqlRegexLexer(**self.options) + + lines = lookahead(line_re.findall(data)) + + # prompt-output cycle + while 1: + + # consume the lines of the command: start with an optional prompt + # and continue until the end of command is detected + curcode = '' + insertions = [] + for line in lines: + # Identify a shell prompt in case of psql commandline example + if line.startswith('$') and not curcode: + lexer = get_lexer_by_name('console', **self.options) + yield from lexer.get_tokens_unprocessed(line) + break + + # Identify a psql prompt + mprompt = re_prompt.match(line) + if mprompt is not None: + insertions.append((len(curcode), + [(0, Generic.Prompt, mprompt.group())])) + curcode += line[len(mprompt.group()):] + else: + curcode += line + + # Check if this is the end of the command + # TODO: better handle multiline comments at the end with + # a lexer with an external state? + if re_psql_command.match(curcode) \ + or re_end_command.search(curcode): + break + + # Emit the combined stream of command and prompt(s) + yield from do_insertions(insertions, + sql.get_tokens_unprocessed(curcode)) + + # Emit the output lines + out_token = Generic.Output + for line in lines: + mprompt = re_prompt.match(line) + if mprompt is not None: + # push the line back to have it processed by the prompt + lines.send(line) + break + + mmsg = re_message.match(line) + if mmsg is not None: + if mmsg.group(1).startswith("ERROR") \ + or mmsg.group(1).startswith("FATAL"): + out_token = Generic.Error + yield (mmsg.start(1), Generic.Strong, mmsg.group(1)) + yield (mmsg.start(2), out_token, mmsg.group(2)) + else: + yield (0, out_token, line) + else: + return + + +class PostgresExplainLexer(RegexLexer): + """ + Handle PostgreSQL EXPLAIN output + + .. versionadded:: 2.15 + """ + + name = 'PostgreSQL EXPLAIN dialect' + aliases = ['postgres-explain'] + filenames = ['*.explain'] + mimetypes = ['text/x-postgresql-explain'] + + tokens = { + 'root': [ + (r'(:|\(|\)|ms|kB|->|\.\.|\,)', Punctuation), + (r'(\s+)', Whitespace), + + # This match estimated cost and effectively measured counters with ANALYZE + # Then, we move to instrumentation state + (r'(cost)(=?)', bygroups(Name.Class, Punctuation), 'instrumentation'), + (r'(actual)( )(=?)', bygroups(Name.Class, Whitespace, Punctuation), 'instrumentation'), + + # Misc keywords + (words(('actual', 'Memory Usage', 'Memory', 'Buckets', 'Batches', + 'originally', 'row', 'rows', 'Hits', 'Misses', + 'Evictions', 'Overflows'), suffix=r'\b'), + Comment.Single), + + (r'(hit|read|dirtied|written|write|time|calls)(=)', bygroups(Comment.Single, Operator)), + (r'(shared|temp|local)', Keyword.Pseudo), + + # We move to sort state in order to emphasize specific keywords (especially disk access) + (r'(Sort Method)(: )', bygroups(Comment.Preproc, Punctuation), 'sort'), + + # These keywords can be followed by an object, like a table + (r'(Sort Key|Group Key|Presorted Key|Hash Key)(:)( )', + bygroups(Comment.Preproc, Punctuation, Whitespace), 'object_name'), + (r'(Cache Key|Cache Mode)(:)( )', bygroups(Comment, Punctuation, Whitespace), 'object_name'), + + # These keywords can be followed by a predicate + (words(('Join Filter', 'Subplans Removed', 'Filter', 'Merge Cond', + 'Hash Cond', 'Index Cond', 'Recheck Cond', 'Heap Blocks', + 'TID Cond', 'Run Condition', 'Order By', 'Function Call', + 'Table Function Call', 'Inner Unique', 'Params Evaluated', + 'Single Copy', 'Sampling', 'One-Time Filter', 'Output', + 'Relations', 'Remote SQL'), suffix=r'\b'), + Comment.Preproc, 'predicate'), + + # Special keyword to handle ON CONFLICT + (r'Conflict ', Comment.Preproc, 'conflict'), + + # Special keyword for InitPlan or SubPlan + (r'(InitPlan|SubPlan)( )(\d+)( )', + bygroups(Keyword, Whitespace, Number.Integer, Whitespace), + 'init_plan'), + + (words(('Sort Method', 'Join Filter', 'Planning time', + 'Planning Time', 'Execution time', 'Execution Time', + 'Workers Planned', 'Workers Launched', 'Buffers', + 'Planning', 'Worker', 'Query Identifier', 'Time', + 'Full-sort Groups', 'Pre-sorted Groups'), suffix=r'\b'), Comment.Preproc), + + # Emphasize these keywords + + (words(('Rows Removed by Join Filter', 'Rows Removed by Filter', + 'Rows Removed by Index Recheck', + 'Heap Fetches', 'never executed'), + suffix=r'\b'), Name.Exception), + (r'(I/O Timings)(:)( )', bygroups(Name.Exception, Punctuation, Whitespace)), + + (words(EXPLAIN_KEYWORDS, suffix=r'\b'), Keyword), + + # join keywords + (r'((Right|Left|Full|Semi|Anti) Join)', Keyword.Type), + (r'(Parallel |Async |Finalize |Partial )', Comment.Preproc), + (r'Backward', Comment.Preproc), + (r'(Intersect|Except|Hash)', Comment.Preproc), + + (r'(CTE)( )(\w*)?', bygroups(Comment, Whitespace, Name.Variable)), + + + # Treat "on" and "using" as a punctuation + (r'(on|using)', Punctuation, 'object_name'), + + + # strings + (r"'(''|[^'])*'", String.Single), + # numbers + (r'\d+\.\d+', Number.Float), + (r'(\d+)', Number.Integer), + + # boolean + (r'(true|false)', Name.Constant), + # explain header + (r'\s*QUERY PLAN\s*\n\s*-+', Comment.Single), + # Settings + (r'(Settings)(:)( )', bygroups(Comment.Preproc, Punctuation, Whitespace), 'setting'), + + # Handle JIT counters + (r'(JIT|Functions|Options|Timing)(:)', bygroups(Comment.Preproc, Punctuation)), + (r'(Inlining|Optimization|Expressions|Deforming|Generation|Emission|Total)', Keyword.Pseudo), + + # Handle Triggers counters + (r'(Trigger)( )(\S*)(:)( )', + bygroups(Comment.Preproc, Whitespace, Name.Variable, Punctuation, Whitespace)), + + ], + 'expression': [ + # matches any kind of parenthesized expression + # the first opening paren is matched by the 'caller' + (r'\(', Punctuation, '#push'), + (r'\)', Punctuation, '#pop'), + (r'(never executed)', Name.Exception), + (r'[^)(]+', Comment), + ], + 'object_name': [ + + # This is a cost or analyze measure + (r'(\(cost)(=?)', bygroups(Name.Class, Punctuation), 'instrumentation'), + (r'(\(actual)( )(=?)', bygroups(Name.Class, Whitespace, Punctuation), 'instrumentation'), + + # if object_name is parenthesized, mark opening paren as + # punctuation, call 'expression', and exit state + (r'\(', Punctuation, 'expression'), + (r'(on)', Punctuation), + # matches possibly schema-qualified table and column names + (r'\w+(\.\w+)*( USING \S+| \w+ USING \S+)', Name.Variable), + (r'\"?\w+\"?(?:\.\"?\w+\"?)?', Name.Variable), + (r'\'\S*\'', Name.Variable), + + # if we encounter a comma, another object is listed + (r',\n', Punctuation, 'object_name'), + (r',', Punctuation, 'object_name'), + + # special case: "*SELECT*" + (r'"\*SELECT\*( \d+)?"(.\w+)?', Name.Variable), + (r'"\*VALUES\*(_\d+)?"(.\w+)?', Name.Variable), + (r'"ANY_subquery"', Name.Variable), + + # Variable $1 ... + (r'\$\d+', Name.Variable), + # cast + (r'::\w+', Name.Variable), + (r' +', Whitespace), + (r'"', Punctuation), + (r'\[\.\.\.\]', Punctuation), + (r'\)', Punctuation, '#pop'), + ], + 'predicate': [ + # if predicate is parenthesized, mark paren as punctuation + (r'(\()([^\n]*)(\))', bygroups(Punctuation, Name.Variable, Punctuation), '#pop'), + # otherwise color until newline + (r'[^\n]*', Name.Variable, '#pop'), + ], + 'instrumentation': [ + (r'=|\.\.', Punctuation), + (r' +', Whitespace), + (r'(rows|width|time|loops)', Name.Class), + (r'\d+\.\d+', Number.Float), + (r'(\d+)', Number.Integer), + (r'\)', Punctuation, '#pop'), + ], + 'conflict': [ + (r'(Resolution: )(\w+)', bygroups(Comment.Preproc, Name.Variable)), + (r'(Arbiter \w+:)', Comment.Preproc, 'object_name'), + (r'(Filter: )', Comment.Preproc, 'predicate'), + ], + 'setting': [ + (r'([a-z_]*?)(\s*)(=)(\s*)(\'.*?\')', bygroups(Name.Attribute, Whitespace, Operator, Whitespace, String)), + (r'\, ', Punctuation), + ], + 'init_plan': [ + (r'\(', Punctuation), + (r'returns \$\d+(,\$\d+)?', Name.Variable), + (r'\)', Punctuation, '#pop'), + ], + 'sort': [ + (r':|kB', Punctuation), + (r'(quicksort|top-N|heapsort|Average|Memory|Peak)', Comment.Prepoc), + (r'(external|merge|Disk|sort)', Name.Exception), + (r'(\d+)', Number.Integer), + (r' +', Whitespace), + ], + } + + +class SqlLexer(RegexLexer): + """ + Lexer for Structured Query Language. Currently, this lexer does + not recognize any special syntax except ANSI SQL. + """ + + name = 'SQL' + aliases = ['sql'] + filenames = ['*.sql'] + mimetypes = ['text/x-sql'] + + flags = re.IGNORECASE + tokens = { + 'root': [ + (r'\s+', Whitespace), + (r'--.*\n?', Comment.Single), + (r'/\*', Comment.Multiline, 'multiline-comments'), + (words(( + 'ABORT', 'ABS', 'ABSOLUTE', 'ACCESS', 'ADA', 'ADD', 'ADMIN', 'AFTER', + 'AGGREGATE', 'ALIAS', 'ALL', 'ALLOCATE', 'ALTER', 'ANALYSE', 'ANALYZE', + 'AND', 'ANY', 'ARE', 'AS', 'ASC', 'ASENSITIVE', 'ASSERTION', 'ASSIGNMENT', + 'ASYMMETRIC', 'AT', 'ATOMIC', 'AUTHORIZATION', 'AVG', 'BACKWARD', + 'BEFORE', 'BEGIN', 'BETWEEN', 'BITVAR', 'BIT_LENGTH', 'BOTH', 'BREADTH', + 'BY', 'C', 'CACHE', 'CALL', 'CALLED', 'CARDINALITY', 'CASCADE', + 'CASCADED', 'CASE', 'CAST', 'CATALOG', 'CATALOG_NAME', 'CHAIN', + 'CHARACTERISTICS', 'CHARACTER_LENGTH', 'CHARACTER_SET_CATALOG', + 'CHARACTER_SET_NAME', 'CHARACTER_SET_SCHEMA', 'CHAR_LENGTH', 'CHECK', + 'CHECKED', 'CHECKPOINT', 'CLASS', 'CLASS_ORIGIN', 'CLOB', 'CLOSE', + 'CLUSTER', 'COALESCE', 'COBOL', 'COLLATE', 'COLLATION', + 'COLLATION_CATALOG', 'COLLATION_NAME', 'COLLATION_SCHEMA', 'COLUMN', + 'COLUMN_NAME', 'COMMAND_FUNCTION', 'COMMAND_FUNCTION_CODE', 'COMMENT', + 'COMMIT', 'COMMITTED', 'COMPLETION', 'CONDITION_NUMBER', 'CONNECT', + 'CONNECTION', 'CONNECTION_NAME', 'CONSTRAINT', 'CONSTRAINTS', + 'CONSTRAINT_CATALOG', 'CONSTRAINT_NAME', 'CONSTRAINT_SCHEMA', + 'CONSTRUCTOR', 'CONTAINS', 'CONTINUE', 'CONVERSION', 'CONVERT', + 'COPY', 'CORRESPONDING', 'COUNT', 'CREATE', 'CREATEDB', 'CREATEUSER', + 'CROSS', 'CUBE', 'CURRENT', 'CURRENT_DATE', 'CURRENT_PATH', + 'CURRENT_ROLE', 'CURRENT_TIME', 'CURRENT_TIMESTAMP', 'CURRENT_USER', + 'CURSOR', 'CURSOR_NAME', 'CYCLE', 'DATA', 'DATABASE', + 'DATETIME_INTERVAL_CODE', 'DATETIME_INTERVAL_PRECISION', 'DAY', + 'DEALLOCATE', 'DECLARE', 'DEFAULT', 'DEFAULTS', 'DEFERRABLE', + 'DEFERRED', 'DEFINED', 'DEFINER', 'DELETE', 'DELIMITER', 'DELIMITERS', + 'DEREF', 'DESC', 'DESCRIBE', 'DESCRIPTOR', 'DESTROY', 'DESTRUCTOR', + 'DETERMINISTIC', 'DIAGNOSTICS', 'DICTIONARY', 'DISCONNECT', 'DISPATCH', + 'DISTINCT', 'DO', 'DOMAIN', 'DROP', 'DYNAMIC', 'DYNAMIC_FUNCTION', + 'DYNAMIC_FUNCTION_CODE', 'EACH', 'ELSE', 'ELSIF', 'ENCODING', + 'ENCRYPTED', 'END', 'END-EXEC', 'EQUALS', 'ESCAPE', 'EVERY', 'EXCEPTION', + 'EXCEPT', 'EXCLUDING', 'EXCLUSIVE', 'EXEC', 'EXECUTE', 'EXISTING', + 'EXISTS', 'EXPLAIN', 'EXTERNAL', 'EXTRACT', 'FALSE', 'FETCH', 'FINAL', + 'FIRST', 'FOR', 'FORCE', 'FOREIGN', 'FORTRAN', 'FORWARD', 'FOUND', 'FREE', + 'FREEZE', 'FROM', 'FULL', 'FUNCTION', 'G', 'GENERAL', 'GENERATED', 'GET', + 'GLOBAL', 'GO', 'GOTO', 'GRANT', 'GRANTED', 'GROUP', 'GROUPING', + 'HANDLER', 'HAVING', 'HIERARCHY', 'HOLD', 'HOST', 'IDENTITY', 'IF', + 'IGNORE', 'ILIKE', 'IMMEDIATE', 'IMMEDIATELY', 'IMMUTABLE', 'IMPLEMENTATION', 'IMPLICIT', + 'IN', 'INCLUDING', 'INCREMENT', 'INDEX', 'INDITCATOR', 'INFIX', + 'INHERITS', 'INITIALIZE', 'INITIALLY', 'INNER', 'INOUT', 'INPUT', + 'INSENSITIVE', 'INSERT', 'INSTANTIABLE', 'INSTEAD', 'INTERSECT', 'INTO', + 'INVOKER', 'IS', 'ISNULL', 'ISOLATION', 'ITERATE', 'JOIN', 'KEY', + 'KEY_MEMBER', 'KEY_TYPE', 'LANCOMPILER', 'LANGUAGE', 'LARGE', 'LAST', + 'LATERAL', 'LEADING', 'LEFT', 'LENGTH', 'LESS', 'LEVEL', 'LIKE', 'LIMIT', + 'LISTEN', 'LOAD', 'LOCAL', 'LOCALTIME', 'LOCALTIMESTAMP', 'LOCATION', + 'LOCATOR', 'LOCK', 'LOWER', 'MAP', 'MATCH', 'MAX', 'MAXVALUE', + 'MESSAGE_LENGTH', 'MESSAGE_OCTET_LENGTH', 'MESSAGE_TEXT', 'METHOD', 'MIN', + 'MINUTE', 'MINVALUE', 'MOD', 'MODE', 'MODIFIES', 'MODIFY', 'MONTH', + 'MORE', 'MOVE', 'MUMPS', 'NAMES', 'NATIONAL', 'NATURAL', 'NCHAR', 'NCLOB', + 'NEW', 'NEXT', 'NO', 'NOCREATEDB', 'NOCREATEUSER', 'NONE', 'NOT', + 'NOTHING', 'NOTIFY', 'NOTNULL', 'NULL', 'NULLABLE', 'NULLIF', 'OBJECT', + 'OCTET_LENGTH', 'OF', 'OFF', 'OFFSET', 'OIDS', 'OLD', 'ON', 'ONLY', + 'OPEN', 'OPERATION', 'OPERATOR', 'OPTION', 'OPTIONS', 'OR', 'ORDER', + 'ORDINALITY', 'OUT', 'OUTER', 'OUTPUT', 'OVERLAPS', 'OVERLAY', + 'OVERRIDING', 'OWNER', 'PAD', 'PARAMETER', 'PARAMETERS', 'PARAMETER_MODE', + 'PARAMETER_NAME', 'PARAMETER_ORDINAL_POSITION', + 'PARAMETER_SPECIFIC_CATALOG', 'PARAMETER_SPECIFIC_NAME', + 'PARAMETER_SPECIFIC_SCHEMA', 'PARTIAL', 'PASCAL', 'PENDANT', 'PERIOD', 'PLACING', + 'PLI', 'POSITION', 'POSTFIX', 'PRECEEDS', 'PRECISION', 'PREFIX', 'PREORDER', + 'PREPARE', 'PRESERVE', 'PRIMARY', 'PRIOR', 'PRIVILEGES', 'PROCEDURAL', + 'PROCEDURE', 'PUBLIC', 'READ', 'READS', 'RECHECK', 'RECURSIVE', 'REF', + 'REFERENCES', 'REFERENCING', 'REINDEX', 'RELATIVE', 'RENAME', + 'REPEATABLE', 'REPLACE', 'RESET', 'RESTART', 'RESTRICT', 'RESULT', + 'RETURN', 'RETURNED_LENGTH', 'RETURNED_OCTET_LENGTH', 'RETURNED_SQLSTATE', + 'RETURNS', 'REVOKE', 'RIGHT', 'ROLE', 'ROLLBACK', 'ROLLUP', 'ROUTINE', + 'ROUTINE_CATALOG', 'ROUTINE_NAME', 'ROUTINE_SCHEMA', 'ROW', 'ROWS', + 'ROW_COUNT', 'RULE', 'SAVE_POINT', 'SCALE', 'SCHEMA', 'SCHEMA_NAME', + 'SCOPE', 'SCROLL', 'SEARCH', 'SECOND', 'SECURITY', 'SELECT', 'SELF', + 'SENSITIVE', 'SERIALIZABLE', 'SERVER_NAME', 'SESSION', 'SESSION_USER', + 'SET', 'SETOF', 'SETS', 'SHARE', 'SHOW', 'SIMILAR', 'SIMPLE', 'SIZE', + 'SOME', 'SOURCE', 'SPACE', 'SPECIFIC', 'SPECIFICTYPE', 'SPECIFIC_NAME', + 'SQL', 'SQLCODE', 'SQLERROR', 'SQLEXCEPTION', 'SQLSTATE', 'SQLWARNINIG', + 'STABLE', 'START', 'STATE', 'STATEMENT', 'STATIC', 'STATISTICS', 'STDIN', + 'STDOUT', 'STORAGE', 'STRICT', 'STRUCTURE', 'STYPE', 'SUBCLASS_ORIGIN', + 'SUBLIST', 'SUBSTRING', 'SUCCEEDS', 'SUM', 'SYMMETRIC', 'SYSID', 'SYSTEM', + 'SYSTEM_USER', 'TABLE', 'TABLE_NAME', ' TEMP', 'TEMPLATE', 'TEMPORARY', + 'TERMINATE', 'THAN', 'THEN', 'TIME', 'TIMESTAMP', 'TIMEZONE_HOUR', + 'TIMEZONE_MINUTE', 'TO', 'TOAST', 'TRAILING', 'TRANSACTION', + 'TRANSACTIONS_COMMITTED', 'TRANSACTIONS_ROLLED_BACK', 'TRANSACTION_ACTIVE', + 'TRANSFORM', 'TRANSFORMS', 'TRANSLATE', 'TRANSLATION', 'TREAT', 'TRIGGER', + 'TRIGGER_CATALOG', 'TRIGGER_NAME', 'TRIGGER_SCHEMA', 'TRIM', 'TRUE', + 'TRUNCATE', 'TRUSTED', 'TYPE', 'UNCOMMITTED', 'UNDER', 'UNENCRYPTED', + 'UNION', 'UNIQUE', 'UNKNOWN', 'UNLISTEN', 'UNNAMED', 'UNNEST', 'UNTIL', + 'UPDATE', 'UPPER', 'USAGE', 'USER', 'USER_DEFINED_TYPE_CATALOG', + 'USER_DEFINED_TYPE_NAME', 'USER_DEFINED_TYPE_SCHEMA', 'USING', 'VACUUM', + 'VALID', 'VALIDATOR', 'VALUES', 'VARIABLE', 'VERBOSE', + 'VERSION', 'VERSIONS', 'VERSIONING', 'VIEW', + 'VOLATILE', 'WHEN', 'WHENEVER', 'WHERE', 'WITH', 'WITHOUT', 'WORK', + 'WRITE', 'YEAR', 'ZONE'), suffix=r'\b'), + Keyword), + (words(( + 'ARRAY', 'BIGINT', 'BINARY', 'BIT', 'BLOB', 'BOOLEAN', 'CHAR', + 'CHARACTER', 'DATE', 'DEC', 'DECIMAL', 'FLOAT', 'INT', 'INTEGER', + 'INTERVAL', 'NUMBER', 'NUMERIC', 'REAL', 'SERIAL', 'SMALLINT', + 'VARCHAR', 'VARYING', 'INT8', 'SERIAL8', 'TEXT'), suffix=r'\b'), + Name.Builtin), + (r'[+*/<>=~!@#%^&|`?-]', Operator), + (r'[0-9]+', Number.Integer), + # TODO: Backslash escapes? + (r"'(''|[^'])*'", String.Single), + (r'"(""|[^"])*"', String.Symbol), # not a real string literal in ANSI SQL + (r'[a-z_][\w$]*', Name), # allow $s in strings for Oracle + (r'[;:()\[\],.]', Punctuation) + ], + 'multiline-comments': [ + (r'/\*', Comment.Multiline, 'multiline-comments'), + (r'\*/', Comment.Multiline, '#pop'), + (r'[^/*]+', Comment.Multiline), + (r'[/*]', Comment.Multiline) + ] + } + + def analyse_text(self, text): + return + + +class TransactSqlLexer(RegexLexer): + """ + Transact-SQL (T-SQL) is Microsoft's and Sybase's proprietary extension to + SQL. + + The list of keywords includes ODBC and keywords reserved for future use.. + """ + + name = 'Transact-SQL' + aliases = ['tsql', 't-sql'] + filenames = ['*.sql'] + mimetypes = ['text/x-tsql'] + + flags = re.IGNORECASE + + tokens = { + 'root': [ + (r'\s+', Whitespace), + (r'--.*?$\n?', Comment.Single), + (r'/\*', Comment.Multiline, 'multiline-comments'), + (words(_tsql_builtins.OPERATORS), Operator), + (words(_tsql_builtins.OPERATOR_WORDS, suffix=r'\b'), Operator.Word), + (words(_tsql_builtins.TYPES, suffix=r'\b'), Name.Class), + (words(_tsql_builtins.FUNCTIONS, suffix=r'\b'), Name.Function), + (r'(goto)(\s+)(\w+\b)', bygroups(Keyword, Whitespace, Name.Label)), + (words(_tsql_builtins.KEYWORDS, suffix=r'\b'), Keyword), + (r'(\[)([^]]+)(\])', bygroups(Operator, Name, Operator)), + (r'0x[0-9a-f]+', Number.Hex), + # Float variant 1, for example: 1., 1.e2, 1.2e3 + (r'[0-9]+\.[0-9]*(e[+-]?[0-9]+)?', Number.Float), + # Float variant 2, for example: .1, .1e2 + (r'\.[0-9]+(e[+-]?[0-9]+)?', Number.Float), + # Float variant 3, for example: 123e45 + (r'[0-9]+e[+-]?[0-9]+', Number.Float), + (r'[0-9]+', Number.Integer), + (r"'(''|[^'])*'", String.Single), + (r'"(""|[^"])*"', String.Symbol), + (r'[;(),.]', Punctuation), + # Below we use \w even for the first "real" character because + # tokens starting with a digit have already been recognized + # as Number above. + (r'@@\w+', Name.Builtin), + (r'@\w+', Name.Variable), + (r'(\w+)(:)', bygroups(Name.Label, Punctuation)), + (r'#?#?\w+', Name), # names for temp tables and anything else + (r'\?', Name.Variable.Magic), # parameter for prepared statements + ], + 'multiline-comments': [ + (r'/\*', Comment.Multiline, 'multiline-comments'), + (r'\*/', Comment.Multiline, '#pop'), + (r'[^/*]+', Comment.Multiline), + (r'[/*]', Comment.Multiline) + ] + } + + def analyse_text(text): + rating = 0 + if tsql_declare_re.search(text): + # Found T-SQL variable declaration. + rating = 1.0 + else: + name_between_backtick_count = len( + name_between_backtick_re.findall(text)) + name_between_bracket_count = len( + name_between_bracket_re.findall(text)) + # We need to check if there are any names using + # backticks or brackets, as otherwise both are 0 + # and 0 >= 2 * 0, so we would always assume it's true + dialect_name_count = name_between_backtick_count + name_between_bracket_count + if dialect_name_count >= 1 and \ + name_between_bracket_count >= 2 * name_between_backtick_count: + # Found at least twice as many [name] as `name`. + rating += 0.5 + elif name_between_bracket_count > name_between_backtick_count: + rating += 0.2 + elif name_between_bracket_count > 0: + rating += 0.1 + if tsql_variable_re.search(text) is not None: + rating += 0.1 + if tsql_go_re.search(text) is not None: + rating += 0.1 + return rating + + +class MySqlLexer(RegexLexer): + """The Oracle MySQL lexer. + + This lexer does not attempt to maintain strict compatibility with + MariaDB syntax or keywords. Although MySQL and MariaDB's common code + history suggests there may be significant overlap between the two, + compatibility between the two is not a target for this lexer. + """ + + name = 'MySQL' + aliases = ['mysql'] + mimetypes = ['text/x-mysql'] + + flags = re.IGNORECASE + tokens = { + 'root': [ + (r'\s+', Whitespace), + + # Comments + (r'(?:#|--\s+).*', Comment.Single), + (r'/\*\+', Comment.Special, 'optimizer-hints'), + (r'/\*', Comment.Multiline, 'multiline-comment'), + + # Hexadecimal literals + (r"x'([0-9a-f]{2})+'", Number.Hex), # MySQL requires paired hex characters in this form. + (r'0x[0-9a-f]+', Number.Hex), + + # Binary literals + (r"b'[01]+'", Number.Bin), + (r'0b[01]+', Number.Bin), + + # Numeric literals + (r'[0-9]+\.[0-9]*(e[+-]?[0-9]+)?', Number.Float), # Mandatory integer, optional fraction and exponent + (r'[0-9]*\.[0-9]+(e[+-]?[0-9]+)?', Number.Float), # Mandatory fraction, optional integer and exponent + (r'[0-9]+e[+-]?[0-9]+', Number.Float), # Exponents with integer significands are still floats + (r'[0-9]+(?=[^0-9a-z$_\u0080-\uffff])', Number.Integer), # Integers that are not in a schema object name + + # Date literals + (r"\{\s*d\s*(?P<quote>['\"])\s*\d{2}(\d{2})?.?\d{2}.?\d{2}\s*(?P=quote)\s*\}", + Literal.Date), + + # Time literals + (r"\{\s*t\s*(?P<quote>['\"])\s*(?:\d+\s+)?\d{1,2}.?\d{1,2}.?\d{1,2}(\.\d*)?\s*(?P=quote)\s*\}", + Literal.Date), + + # Timestamp literals + ( + r"\{\s*ts\s*(?P<quote>['\"])\s*" + r"\d{2}(?:\d{2})?.?\d{2}.?\d{2}" # Date part + r"\s+" # Whitespace between date and time + r"\d{1,2}.?\d{1,2}.?\d{1,2}(\.\d*)?" # Time part + r"\s*(?P=quote)\s*\}", + Literal.Date + ), + + # String literals + (r"'", String.Single, 'single-quoted-string'), + (r'"', String.Double, 'double-quoted-string'), + + # Variables + (r'@@(?:global\.|persist\.|persist_only\.|session\.)?[a-z_]+', Name.Variable), + (r'@[a-z0-9_$.]+', Name.Variable), + (r"@'", Name.Variable, 'single-quoted-variable'), + (r'@"', Name.Variable, 'double-quoted-variable'), + (r"@`", Name.Variable, 'backtick-quoted-variable'), + (r'\?', Name.Variable), # For demonstrating prepared statements + + # Operators + (r'[!%&*+/:<=>^|~-]+', Operator), + + # Exceptions; these words tokenize differently in different contexts. + (r'\b(set)(?!\s*\()', Keyword), + (r'\b(character)(\s+)(set)\b', bygroups(Keyword, Whitespace, Keyword)), + # In all other known cases, "SET" is tokenized by MYSQL_DATATYPES. + + (words(MYSQL_CONSTANTS, prefix=r'\b', suffix=r'\b'), Name.Constant), + (words(MYSQL_DATATYPES, prefix=r'\b', suffix=r'\b'), Keyword.Type), + (words(MYSQL_KEYWORDS, prefix=r'\b', suffix=r'\b'), Keyword), + (words(MYSQL_FUNCTIONS, prefix=r'\b', suffix=r'\b(\s*)(\()'), + bygroups(Name.Function, Whitespace, Punctuation)), + + # Schema object names + # + # Note: Although the first regex supports unquoted all-numeric + # identifiers, this will not be a problem in practice because + # numeric literals have already been handled above. + # + ('[0-9a-z$_\u0080-\uffff]+', Name), + (r'`', Name.Quoted, 'schema-object-name'), + + # Punctuation + (r'[(),.;]', Punctuation), + ], + + # Multiline comment substates + # --------------------------- + + 'optimizer-hints': [ + (r'[^*a-z]+', Comment.Special), + (r'\*/', Comment.Special, '#pop'), + (words(MYSQL_OPTIMIZER_HINTS, suffix=r'\b'), Comment.Preproc), + ('[a-z]+', Comment.Special), + (r'\*', Comment.Special), + ], + + 'multiline-comment': [ + (r'[^*]+', Comment.Multiline), + (r'\*/', Comment.Multiline, '#pop'), + (r'\*', Comment.Multiline), + ], + + # String substates + # ---------------- + + 'single-quoted-string': [ + (r"[^'\\]+", String.Single), + (r"''", String.Escape), + (r"""\\[0'"bnrtZ\\%_]""", String.Escape), + (r"'", String.Single, '#pop'), + ], + + 'double-quoted-string': [ + (r'[^"\\]+', String.Double), + (r'""', String.Escape), + (r"""\\[0'"bnrtZ\\%_]""", String.Escape), + (r'"', String.Double, '#pop'), + ], + + # Variable substates + # ------------------ + + 'single-quoted-variable': [ + (r"[^']+", Name.Variable), + (r"''", Name.Variable), + (r"'", Name.Variable, '#pop'), + ], + + 'double-quoted-variable': [ + (r'[^"]+', Name.Variable), + (r'""', Name.Variable), + (r'"', Name.Variable, '#pop'), + ], + + 'backtick-quoted-variable': [ + (r'[^`]+', Name.Variable), + (r'``', Name.Variable), + (r'`', Name.Variable, '#pop'), + ], + + # Schema object name substates + # ---------------------------- + # + # "Name.Quoted" and "Name.Quoted.Escape" are non-standard but + # formatters will style them as "Name" by default but add + # additional styles based on the token name. This gives users + # flexibility to add custom styles as desired. + # + 'schema-object-name': [ + (r'[^`]+', Name.Quoted), + (r'``', Name.Quoted.Escape), + (r'`', Name.Quoted, '#pop'), + ], + } + + def analyse_text(text): + rating = 0 + name_between_backtick_count = len( + name_between_backtick_re.findall(text)) + name_between_bracket_count = len( + name_between_bracket_re.findall(text)) + # Same logic as above in the TSQL analysis + dialect_name_count = name_between_backtick_count + name_between_bracket_count + if dialect_name_count >= 1 and \ + name_between_backtick_count >= 2 * name_between_bracket_count: + # Found at least twice as many `name` as [name]. + rating += 0.5 + elif name_between_backtick_count > name_between_bracket_count: + rating += 0.2 + elif name_between_backtick_count > 0: + rating += 0.1 + return rating + + +class SqliteConsoleLexer(Lexer): + """ + Lexer for example sessions using sqlite3. + + .. versionadded:: 0.11 + """ + + name = 'sqlite3con' + aliases = ['sqlite3'] + filenames = ['*.sqlite3-console'] + mimetypes = ['text/x-sqlite3-console'] + + def get_tokens_unprocessed(self, data): + sql = SqlLexer(**self.options) + + curcode = '' + insertions = [] + for match in line_re.finditer(data): + line = match.group() + prompt_match = sqlite_prompt_re.match(line) + if prompt_match is not None: + insertions.append((len(curcode), + [(0, Generic.Prompt, line[:7])])) + insertions.append((len(curcode), + [(7, Whitespace, ' ')])) + curcode += line[8:] + else: + if curcode: + yield from do_insertions(insertions, + sql.get_tokens_unprocessed(curcode)) + curcode = '' + insertions = [] + if line.startswith('SQL error: '): + yield (match.start(), Generic.Traceback, line) + else: + yield (match.start(), Generic.Output, line) + if curcode: + yield from do_insertions(insertions, + sql.get_tokens_unprocessed(curcode)) + + +class RqlLexer(RegexLexer): + """ + Lexer for Relation Query Language. + + .. versionadded:: 2.0 + """ + name = 'RQL' + url = 'http://www.logilab.org/project/rql' + aliases = ['rql'] + filenames = ['*.rql'] + mimetypes = ['text/x-rql'] + + flags = re.IGNORECASE + tokens = { + 'root': [ + (r'\s+', Whitespace), + (r'(DELETE|SET|INSERT|UNION|DISTINCT|WITH|WHERE|BEING|OR' + r'|AND|NOT|GROUPBY|HAVING|ORDERBY|ASC|DESC|LIMIT|OFFSET' + r'|TODAY|NOW|TRUE|FALSE|NULL|EXISTS)\b', Keyword), + (r'[+*/<>=%-]', Operator), + (r'(Any|is|instance_of|CWEType|CWRelation)\b', Name.Builtin), + (r'[0-9]+', Number.Integer), + (r'[A-Z_]\w*\??', Name), + (r"'(''|[^'])*'", String.Single), + (r'"(""|[^"])*"', String.Single), + (r'[;:()\[\],.]', Punctuation) + ], + } |