Files
2026-09-05 23:43:46 +03:00

1112 lines
34 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#
# Copyright (C) 2009-2020 the sqlparse authors and contributors
# <see AUTHORS file>
#
# This module is part of python-sqlparse and is released under
# the BSD License: https://opensource.org/licenses/BSD-3-Clause
import re
from bisect import bisect_left
from collections import defaultdict
from sqlparse import tokens
# object() only supports "is" and is useful as a marker
# use this marker to specify that the given regex in SQL_REGEX
# shall be processed further through a lookup in the KEYWORDS dictionaries
PROCESS_AS_KEYWORD = object()
# Dollar-quoted literals (`$tag$...$tag$`) and multiline comments
# (`/*...*/`, `/*+...*/`) used to be matched with per-position regexes
# using a lazy dot-all quantifier (`[\s\S]*?`) terminated by a
# backreference or a literal delimiter. Applied at every text position by
# the lexer loop, that shape is O(n^2) on adversarial input with many
# unclosed openers, since each failed attempt re-scans to the end of the
# remaining text (GHSA-prg7-hcfm-mfcr). find_delimited_spans() collects
# all opener and closer positions in a single pass instead, so the lexer
# can pair them with a binary search when it actually reaches an opener.
#
# The delimiter regex is wrapped in a lookahead so that overlapping
# occurrences are reported too: in `$$$$` the closing `$$` starts inside
# the match of the opening one, and a non-overlapping scan would miss it.
_DOLLAR_QUOTE_DELIM = re.compile(
r'(?=(\$(?:[_A-ZÀ-Ü]\w*)?\$))', re.IGNORECASE | re.UNICODE)
_DOLLAR_QUOTE_OPENER_OK = re.compile(r'(?<![\w"$])', re.UNICODE)
_COMMENT_HINT_OPEN = re.compile(r'/\*\+')
_COMMENT_OPEN = re.compile(r'/\*(?!\+)')
_COMMENT_CLOSE = re.compile(r'\*/')
# All multiline comments share a single closer (`*/`), so one bucket is
# enough; dollar-quoted literals are bucketed by their tag.
_COMMENT_TAG = '/*'
class DelimitedSpans:
"""Opener and closer positions of dollar-quoted literals and multiline
comments, resolved on demand by :meth:`resolve`.
Pairing has to happen at the position the lexer has actually reached,
not upfront: a `/*`, `*/` or `$$` inside a string literal or behind a
`--` comment is not a delimiter at all, and resolving it eagerly would
consume the partner of the next real delimiter.
"""
__slots__ = ('_closers', 'openers')
def __init__(self, openers, closers):
#: start offset -> (offset behind the opener, tag, token type)
self.openers = openers
#: tag -> ([closer start, ...], [closer end, ...]), both ascending
self._closers = closers
def resolve(self, pos):
"""Return ``(end offset, token type)`` for the span opening at
`pos`, or None if its closing delimiter is missing -- just as a
regex that never finds its closing delimiter fails to match.
"""
opener_end, tag, ttype = self.openers[pos]
starts, ends = self._closers[tag]
idx = bisect_left(starts, opener_end)
if idx == len(starts):
return None
return ends[idx], ttype
_NO_SPANS = DelimitedSpans({}, {})
def find_delimited_spans(text):
"""Locate the delimiters of dollar-quoted literals and multiline
comments in `text` and return them as a :class:`DelimitedSpans`.
"""
has_dollar = '$' in text
# A comment can only ever open on "/*"; without it "*/" alone can
# never pair with anything, so gating on "/*" alone is sufficient to
# skip all three comment-related regex passes below.
has_comment_open = '/*' in text
if not has_dollar and not has_comment_open:
return _NO_SPANS
openers = {}
closers = defaultdict(lambda: ([], []))
if has_dollar:
for m in _DOLLAR_QUOTE_DELIM.finditer(text):
start = m.start()
tag = m.group(1)
end = start + len(tag)
# Every delimiter can close a literal, but one preceded by a
# word character, `"` or `$` cannot open one.
starts, ends = closers[tag]
starts.append(start)
ends.append(end)
if _DOLLAR_QUOTE_OPENER_OK.match(text, start) is not None:
openers[start] = (end, tag, tokens.Literal)
if has_comment_open:
for m in _COMMENT_HINT_OPEN.finditer(text):
openers[m.start()] = (
m.end(), _COMMENT_TAG, tokens.Comment.Multiline.Hint)
for m in _COMMENT_OPEN.finditer(text):
openers[m.start()] = (
m.end(), _COMMENT_TAG, tokens.Comment.Multiline)
starts, ends = closers[_COMMENT_TAG]
for m in _COMMENT_CLOSE.finditer(text):
starts.append(m.start())
ends.append(m.end())
return DelimitedSpans(openers, closers)
SQL_REGEX = [
(r'(--|# )\+.*?(\r\n|\r|\n|$)', tokens.Comment.Single.Hint),
(r'(--|# ).*?(\r\n|\r|\n|$)', tokens.Comment.Single),
(r'(\r\n|\r|\n)', tokens.Newline),
(r'\s+?', tokens.Whitespace),
(r':=', tokens.Assignment),
(r'::', tokens.Punctuation),
(r'\*', tokens.Wildcard),
(r"`(``|[^`])*`", tokens.Name),
(r"´(´´|[^´])*´", tokens.Name),
(r'\?', tokens.Name.Placeholder),
(r'%(\(\w+\))?s', tokens.Name.Placeholder),
(r'(?<!\w)[$:?]\w+', tokens.Name.Placeholder),
(r'\\\w+', tokens.Command),
# FIXME(andi): VALUES shouldn't be listed here
# see https://github.com/andialbrecht/sqlparse/pull/64
# AS and IN are special, it may be followed by a parenthesis, but
# are never functions, see issue183 and issue507
(r'(CASE|IN|VALUES|USING|FROM|AS)\b', tokens.Keyword),
(r'(@|##|#)[A-ZÀ-Ü]\w+', tokens.Name),
# see issue #39
# Spaces around period `schema . name` are valid identifier
# TODO: Spaces before period not implemented
# The negative lookahead ``(?!\d)`` keeps a following floating point
# literal such as ``.03`` from being mistaken for a member access, so a
# preceding keyword (e.g. ``BETWEEN``) is not reclassified as a name.
# See issue #601.
(r'[A-ZÀ-Ü]\w*(?=\s*\.(?!\d))', tokens.Name), # 'Name'.
# FIXME(atronah): never match,
# because `re.match` doesn't work with look-behind regexp feature
(r'(?<=\.)[A-ZÀ-Ü]\w*', tokens.Name), # .'Name'
(r'[A-ZÀ-Ü]\w*(?=\()', tokens.Name), # side effect: change kw to func
(r'-?0x[\dA-F]+', tokens.Number.Hexadecimal),
(r'-?\d+(\.\d+)?E-?\d+', tokens.Number.Float),
(r'(?![_A-ZÀ-Ü])-?(\d+(\.\d*)|\.\d+)(?![_A-ZÀ-Ü])',
tokens.Number.Float),
(r'(?![_A-ZÀ-Ü])-?\d+(?![_A-ZÀ-Ü])', tokens.Number.Integer),
(r"'(''|\\'|[^'])*'", tokens.String.Single),
# not a real string literal in ANSI SQL:
(r'"(""|\\"|[^"])*"', tokens.String.Symbol),
(r'(""|".*?[^\\]")', tokens.String.Symbol),
# sqlite names can be escaped with [square brackets]. left bracket
# cannot be preceded by word character or a right bracket --
# otherwise it's probably an array index
(r'(?<![\w\])])(\[[^\]\[]+\])', tokens.Name),
(r'((LEFT\s+|RIGHT\s+|FULL\s+)?(INNER\s+|OUTER\s+|STRAIGHT\s+)?'
r'|(CROSS\s+|NATURAL\s+)?)?JOIN\b', tokens.Keyword),
(r'END(\s+IF|\s+LOOP|\s+WHILE|\s+FOR|\s+CASE)?\b', tokens.Keyword),
(r'IF\s+(NOT\s+)?EXISTS\b', tokens.Keyword),
(r'NOT\s+NULL\b', tokens.Keyword),
(r'(ASC|DESC)(\s+NULLS\s+(FIRST|LAST))?\b', tokens.Keyword.Order),
(r'(ASC|DESC)\b', tokens.Keyword.Order),
(r'NULLS\s+(FIRST|LAST)\b', tokens.Keyword.Order),
(r'UNION\s+ALL\b', tokens.Keyword),
(r'CREATE(\s+OR\s+REPLACE)?\b', tokens.Keyword.DDL),
(r'DOUBLE\s+PRECISION\b', tokens.Name.Builtin),
(r'GROUP\s+BY\b', tokens.Keyword),
(r'ORDER\s+BY\b', tokens.Keyword),
(r'PRIMARY\s+KEY\b', tokens.Keyword),
(r'HANDLER\s+FOR\b', tokens.Keyword),
(r'GO(\s\d+)\b', tokens.Keyword),
(r'(LATERAL\s+VIEW\s+)'
r'(EXPLODE|INLINE|PARSE_URL_TUPLE|POSEXPLODE|STACK)\b',
tokens.Keyword),
(r"(AT|WITH')\s+TIME\s+ZONE\s+'[^']+'", tokens.Keyword.TZCast),
(r'(NOT\s+)?(LIKE|ILIKE|RLIKE)\b', tokens.Operator.Comparison),
(r'(NOT\s+)?(REGEXP)(\s+(BINARY))?\b', tokens.Operator.Comparison),
# Check for keywords, also returns tokens.Name if regex matches
# but the match isn't a keyword.
(r'\w[$#\w]*', PROCESS_AS_KEYWORD),
(r'[;:()\[\],\.]', tokens.Punctuation),
# JSON operators
(r'(\->>?|#>>?|@>|<@|\?\|?|\?&|\-|#\-)', tokens.Operator),
(r'[<>=~!]+', tokens.Operator.Comparison),
(r'[+/@#%^&|^-]+', tokens.Operator),
]
KEYWORDS = {
'ABORT': tokens.Keyword,
'ABS': tokens.Keyword,
'ABSOLUTE': tokens.Keyword,
'ACCESS': tokens.Keyword,
'ADA': tokens.Keyword,
'ADD': tokens.Keyword,
'ADMIN': tokens.Keyword,
'AFTER': tokens.Keyword,
'AGGREGATE': tokens.Keyword,
'ALIAS': tokens.Keyword,
'ALL': tokens.Keyword,
'ALLOCATE': tokens.Keyword,
'ANALYSE': tokens.Keyword,
'ANALYZE': tokens.Keyword,
'ANY': tokens.Keyword,
'ARRAYLEN': tokens.Keyword,
'ARE': tokens.Keyword,
'ASENSITIVE': tokens.Keyword,
'ASSERTION': tokens.Keyword,
'ASSIGNMENT': tokens.Keyword,
'ASYMMETRIC': tokens.Keyword,
'AT': tokens.Keyword,
'ATOMIC': tokens.Keyword,
'AUDIT': tokens.Keyword,
'AUTHORIZATION': tokens.Keyword,
'AUTO_INCREMENT': tokens.Keyword,
'AVG': tokens.Keyword,
'BACKWARD': tokens.Keyword,
'BEFORE': tokens.Keyword,
'BEGIN': tokens.Keyword,
'BETWEEN': tokens.Keyword,
'BITVAR': tokens.Keyword,
'BIT_LENGTH': tokens.Keyword,
'BOTH': tokens.Keyword,
'BREADTH': tokens.Keyword,
# 'C': tokens.Keyword, # most likely this is an alias
'CACHE': tokens.Keyword,
'CALL': tokens.Keyword,
'CALLED': tokens.Keyword,
'CARDINALITY': tokens.Keyword,
'CASCADE': tokens.Keyword,
'CASCADED': tokens.Keyword,
'CAST': tokens.Keyword,
'CATALOG': tokens.Keyword,
'CATALOG_NAME': tokens.Keyword,
'CHAIN': tokens.Keyword,
'CHARACTERISTICS': tokens.Keyword,
'CHARACTER_LENGTH': tokens.Keyword,
'CHARACTER_SET_CATALOG': tokens.Keyword,
'CHARACTER_SET_NAME': tokens.Keyword,
'CHARACTER_SET_SCHEMA': tokens.Keyword,
'CHAR_LENGTH': tokens.Keyword,
'CHARSET': tokens.Keyword,
'CHECK': tokens.Keyword,
'CHECKED': tokens.Keyword,
'CHECKPOINT': tokens.Keyword,
'CLASS': tokens.Keyword,
'CLASS_ORIGIN': tokens.Keyword,
'CLOB': tokens.Keyword,
'CLOSE': tokens.Keyword,
'CLUSTER': tokens.Keyword,
'COALESCE': tokens.Keyword,
'COBOL': tokens.Keyword,
'COLLATE': tokens.Keyword,
'COLLATION': tokens.Keyword,
'COLLATION_CATALOG': tokens.Keyword,
'COLLATION_NAME': tokens.Keyword,
'COLLATION_SCHEMA': tokens.Keyword,
'COLLECT': tokens.Keyword,
'COLUMN': tokens.Keyword,
'COLUMN_NAME': tokens.Keyword,
'COMPRESS': tokens.Keyword,
'COMMAND_FUNCTION': tokens.Keyword,
'COMMAND_FUNCTION_CODE': tokens.Keyword,
'COMMENT': tokens.Keyword,
'COMMIT': tokens.Keyword.DML,
'COMMITTED': tokens.Keyword,
'COMPLETION': tokens.Keyword,
'CONCURRENTLY': tokens.Keyword,
'CONDITION_NUMBER': tokens.Keyword,
'CONNECT': tokens.Keyword,
'CONNECTION': tokens.Keyword,
'CONNECTION_NAME': tokens.Keyword,
'CONSTRAINT': tokens.Keyword,
'CONSTRAINTS': tokens.Keyword,
'CONSTRAINT_CATALOG': tokens.Keyword,
'CONSTRAINT_NAME': tokens.Keyword,
'CONSTRAINT_SCHEMA': tokens.Keyword,
'CONSTRUCTOR': tokens.Keyword,
'CONTAINS': tokens.Keyword,
'CONTINUE': tokens.Keyword,
'CONVERSION': tokens.Keyword,
'CONVERT': tokens.Keyword,
'COPY': tokens.Keyword,
'CORRESPONDING': tokens.Keyword,
'COUNT': tokens.Keyword,
'CREATEDB': tokens.Keyword,
'CREATEUSER': tokens.Keyword,
'CROSS': tokens.Keyword,
'CUBE': tokens.Keyword,
'CURRENT': tokens.Keyword,
'CURRENT_DATE': tokens.Keyword,
'CURRENT_PATH': tokens.Keyword,
'CURRENT_ROLE': tokens.Keyword,
'CURRENT_TIME': tokens.Keyword,
'CURRENT_TIMESTAMP': tokens.Keyword,
'CURRENT_USER': tokens.Keyword,
'CURSOR': tokens.Keyword,
'CURSOR_NAME': tokens.Keyword,
'CYCLE': tokens.Keyword,
'DATA': tokens.Keyword,
'DATABASE': tokens.Keyword,
'DATETIME_INTERVAL_CODE': tokens.Keyword,
'DATETIME_INTERVAL_PRECISION': tokens.Keyword,
'DAY': tokens.Keyword,
'DEALLOCATE': tokens.Keyword,
'DECLARE': tokens.Keyword,
'DEFAULT': tokens.Keyword,
'DEFAULTS': tokens.Keyword,
'DEFERRABLE': tokens.Keyword,
'DEFERRED': tokens.Keyword,
'DEFINED': tokens.Keyword,
'DEFINER': tokens.Keyword,
'DELIMITER': tokens.Keyword,
'DELIMITERS': tokens.Keyword,
'DEREF': tokens.Keyword,
'DESCRIBE': tokens.Keyword,
'DESCRIPTOR': tokens.Keyword,
'DESTROY': tokens.Keyword,
'DESTRUCTOR': tokens.Keyword,
'DETERMINISTIC': tokens.Keyword,
'DIAGNOSTICS': tokens.Keyword,
'DICTIONARY': tokens.Keyword,
'DISABLE': tokens.Keyword,
'DISCONNECT': tokens.Keyword,
'DISPATCH': tokens.Keyword,
'DIV': tokens.Operator,
'DO': tokens.Keyword,
'DOMAIN': tokens.Keyword,
'DYNAMIC': tokens.Keyword,
'DYNAMIC_FUNCTION': tokens.Keyword,
'DYNAMIC_FUNCTION_CODE': tokens.Keyword,
'EACH': tokens.Keyword,
'ENABLE': tokens.Keyword,
'ENCODING': tokens.Keyword,
'ENCRYPTED': tokens.Keyword,
'END-EXEC': tokens.Keyword,
'ENGINE': tokens.Keyword,
'EQUALS': tokens.Keyword,
'ESCAPE': tokens.Keyword,
'EVERY': tokens.Keyword,
'EXCEPT': tokens.Keyword,
'EXCEPTION': tokens.Keyword,
'EXCLUDING': tokens.Keyword,
'EXCLUSIVE': tokens.Keyword,
'EXEC': tokens.Keyword,
'EXECUTE': tokens.Keyword,
'EXISTING': tokens.Keyword,
'EXISTS': tokens.Keyword,
'EXPLAIN': tokens.Keyword,
'EXTERNAL': tokens.Keyword,
'EXTRACT': tokens.Keyword,
'FALSE': tokens.Keyword,
'FETCH': tokens.Keyword,
'FILE': tokens.Keyword,
'FINAL': tokens.Keyword,
'FIRST': tokens.Keyword,
'FORCE': tokens.Keyword,
'FOREACH': tokens.Keyword,
'FOREIGN': tokens.Keyword,
'FORTRAN': tokens.Keyword,
'FORWARD': tokens.Keyword,
'FOUND': tokens.Keyword,
'FREE': tokens.Keyword,
'FREEZE': tokens.Keyword,
'FULL': tokens.Keyword,
'FUNCTION': tokens.Keyword,
# 'G': tokens.Keyword,
'GENERAL': tokens.Keyword,
'GENERATED': tokens.Keyword,
'GET': tokens.Keyword,
'GLOBAL': tokens.Keyword,
'GO': tokens.Keyword,
'GOTO': tokens.Keyword,
'GRANTED': tokens.Keyword,
'GROUPING': tokens.Keyword,
'HAVING': tokens.Keyword,
'HIERARCHY': tokens.Keyword,
'HOLD': tokens.Keyword,
'HOUR': tokens.Keyword,
'HOST': tokens.Keyword,
'IDENTIFIED': tokens.Keyword,
'IDENTITY': tokens.Keyword,
'IGNORE': tokens.Keyword,
'ILIKE': tokens.Keyword,
'IMMEDIATE': tokens.Keyword,
'IMMUTABLE': tokens.Keyword,
'IMPLEMENTATION': tokens.Keyword,
'IMPLICIT': tokens.Keyword,
'INCLUDING': tokens.Keyword,
'INCREMENT': tokens.Keyword,
'INDEX': tokens.Keyword,
'INDICATOR': tokens.Keyword,
'INFIX': tokens.Keyword,
'INHERITS': tokens.Keyword,
'INITIAL': tokens.Keyword,
'INITIALIZE': tokens.Keyword,
'INITIALLY': tokens.Keyword,
'INOUT': tokens.Keyword,
'INPUT': tokens.Keyword,
'INSENSITIVE': tokens.Keyword,
'INSTANTIABLE': tokens.Keyword,
'INSTEAD': tokens.Keyword,
'INTERSECT': tokens.Keyword,
'INTO': tokens.Keyword,
'INVOKER': tokens.Keyword,
'IS': tokens.Keyword,
'ISNULL': tokens.Keyword,
'ISOLATION': tokens.Keyword,
'ITERATE': tokens.Keyword,
# 'K': tokens.Keyword,
'KEY': tokens.Keyword,
'KEY_MEMBER': tokens.Keyword,
'KEY_TYPE': tokens.Keyword,
'LANCOMPILER': tokens.Keyword,
'LANGUAGE': tokens.Keyword,
'LARGE': tokens.Keyword,
'LAST': tokens.Keyword,
'LATERAL': tokens.Keyword,
'LEADING': tokens.Keyword,
'LENGTH': tokens.Keyword,
'LESS': tokens.Keyword,
'LEVEL': tokens.Keyword,
'LIMIT': tokens.Keyword,
'LISTEN': tokens.Keyword,
'LOAD': tokens.Keyword,
'LOCAL': tokens.Keyword,
'LOCALTIME': tokens.Keyword,
'LOCALTIMESTAMP': tokens.Keyword,
'LOCATION': tokens.Keyword,
'LOCATOR': tokens.Keyword,
'LOCK': tokens.Keyword,
'LOWER': tokens.Keyword,
# 'M': tokens.Keyword,
'MAP': tokens.Keyword,
'MATCH': tokens.Keyword,
'MATERIALIZED': tokens.Keyword,
'MAXEXTENTS': tokens.Keyword,
'MAXVALUE': tokens.Keyword,
'MESSAGE_LENGTH': tokens.Keyword,
'MESSAGE_OCTET_LENGTH': tokens.Keyword,
'MESSAGE_TEXT': tokens.Keyword,
'METHOD': tokens.Keyword,
'MINUTE': tokens.Keyword,
'MINUS': tokens.Keyword,
'MINVALUE': tokens.Keyword,
'MOD': tokens.Keyword,
'MODE': tokens.Keyword,
'MODIFIES': tokens.Keyword,
'MODIFY': tokens.Keyword,
'MONTH': tokens.Keyword,
'MORE': tokens.Keyword,
'MOVE': tokens.Keyword,
'MUMPS': tokens.Keyword,
'NAMES': tokens.Keyword,
'NATIONAL': tokens.Keyword,
'NATURAL': tokens.Keyword,
'NCHAR': tokens.Keyword,
'NCLOB': tokens.Keyword,
'NEW': tokens.Keyword,
'NEXT': tokens.Keyword,
'NO': tokens.Keyword,
'NOAUDIT': tokens.Keyword,
'NOCOMPRESS': tokens.Keyword,
'NOCREATEDB': tokens.Keyword,
'NOCREATEUSER': tokens.Keyword,
'NONE': tokens.Keyword,
'NOT': tokens.Keyword,
'NOTFOUND': tokens.Keyword,
'NOTHING': tokens.Keyword,
'NOTIFY': tokens.Keyword,
'NOTNULL': tokens.Keyword,
'NOWAIT': tokens.Keyword,
'NULL': tokens.Keyword,
'NULLABLE': tokens.Keyword,
'NULLIF': tokens.Keyword,
'OBJECT': tokens.Keyword,
'OCTET_LENGTH': tokens.Keyword,
'OF': tokens.Keyword,
'OFF': tokens.Keyword,
'OFFLINE': tokens.Keyword,
'OFFSET': tokens.Keyword,
'OIDS': tokens.Keyword,
'OLD': tokens.Keyword,
'ONLINE': tokens.Keyword,
'ONLY': tokens.Keyword,
'OPEN': tokens.Keyword,
'OPERATION': tokens.Keyword,
'OPERATOR': tokens.Keyword,
'OPTION': tokens.Keyword,
'OPTIONS': tokens.Keyword,
'ORDINALITY': tokens.Keyword,
'OUT': tokens.Keyword,
'OUTPUT': tokens.Keyword,
'OVERLAPS': tokens.Keyword,
'OVERLAY': tokens.Keyword,
'OVERRIDING': tokens.Keyword,
'OWNER': tokens.Keyword,
'QUARTER': tokens.Keyword,
'PAD': tokens.Keyword,
'PARAMETER': tokens.Keyword,
'PARAMETERS': tokens.Keyword,
'PARAMETER_MODE': tokens.Keyword,
'PARAMETER_NAME': tokens.Keyword,
'PARAMETER_ORDINAL_POSITION': tokens.Keyword,
'PARAMETER_SPECIFIC_CATALOG': tokens.Keyword,
'PARAMETER_SPECIFIC_NAME': tokens.Keyword,
'PARAMETER_SPECIFIC_SCHEMA': tokens.Keyword,
'PARTIAL': tokens.Keyword,
'PASCAL': tokens.Keyword,
'PCTFREE': tokens.Keyword,
'PENDANT': tokens.Keyword,
'PLACING': tokens.Keyword,
'PLI': tokens.Keyword,
'POSITION': tokens.Keyword,
'POSTFIX': tokens.Keyword,
'PRECISION': tokens.Keyword,
'PREFIX': tokens.Keyword,
'PREORDER': tokens.Keyword,
'PREPARE': tokens.Keyword,
'PRESERVE': tokens.Keyword,
'PRIMARY': tokens.Keyword,
'PRIOR': tokens.Keyword,
'PRIVILEGES': tokens.Keyword,
'PROCEDURAL': tokens.Keyword,
'PROCEDURE': tokens.Keyword,
'PUBLIC': tokens.Keyword,
'RAISE': tokens.Keyword,
'RAW': tokens.Keyword,
'READ': tokens.Keyword,
'READS': tokens.Keyword,
'RECHECK': tokens.Keyword,
'RECURSIVE': tokens.Keyword,
'REF': tokens.Keyword,
'REFERENCES': tokens.Keyword,
'REFERENCING': tokens.Keyword,
'REINDEX': tokens.Keyword,
'RELATIVE': tokens.Keyword,
'RENAME': tokens.Keyword,
'REPEATABLE': tokens.Keyword,
'RESET': tokens.Keyword,
'RESOURCE': tokens.Keyword,
'RESTART': tokens.Keyword,
'RESTRICT': tokens.Keyword,
'RESULT': tokens.Keyword,
'RETURN': tokens.Keyword,
'RETURNED_LENGTH': tokens.Keyword,
'RETURNED_OCTET_LENGTH': tokens.Keyword,
'RETURNED_SQLSTATE': tokens.Keyword,
'RETURNING': tokens.Keyword,
'RETURNS': tokens.Keyword,
'RIGHT': tokens.Keyword,
'ROLE': tokens.Keyword,
'ROLLBACK': tokens.Keyword.DML,
'ROLLUP': tokens.Keyword,
'ROUTINE': tokens.Keyword,
'ROUTINE_CATALOG': tokens.Keyword,
'ROUTINE_NAME': tokens.Keyword,
'ROUTINE_SCHEMA': tokens.Keyword,
'ROWS': tokens.Keyword,
'ROW_COUNT': tokens.Keyword,
'ROW_FORMAT': tokens.Keyword,
'RULE': tokens.Keyword,
'SAVE_POINT': tokens.Keyword,
'SCALE': tokens.Keyword,
'SCHEMA': tokens.Keyword,
'SCHEMA_NAME': tokens.Keyword,
'SCOPE': tokens.Keyword,
'SCROLL': tokens.Keyword,
'SEARCH': tokens.Keyword,
'SECOND': tokens.Keyword,
'SECURITY': tokens.Keyword,
'SELF': tokens.Keyword,
'SENSITIVE': tokens.Keyword,
'SEQUENCE': tokens.Keyword,
'SERIALIZABLE': tokens.Keyword,
'SERVER_NAME': tokens.Keyword,
'SESSION': tokens.Keyword,
'SESSION_USER': tokens.Keyword,
'SETOF': tokens.Keyword,
'SETS': tokens.Keyword,
'SHARE': tokens.Keyword,
'SHOW': tokens.Keyword,
'SIMILAR': tokens.Keyword,
'SIMPLE': tokens.Keyword,
'SIZE': tokens.Keyword,
'SOME': tokens.Keyword,
'SOURCE': tokens.Keyword,
'SPACE': tokens.Keyword,
'SPECIFIC': tokens.Keyword,
'SPECIFICTYPE': tokens.Keyword,
'SPECIFIC_NAME': tokens.Keyword,
'SQL': tokens.Keyword,
'SQLBUF': tokens.Keyword,
'SQLCODE': tokens.Keyword,
'SQLERROR': tokens.Keyword,
'SQLEXCEPTION': tokens.Keyword,
'SQLSTATE': tokens.Keyword,
'SQLWARNING': tokens.Keyword,
'STABLE': tokens.Keyword,
'START': tokens.Keyword.DML,
# 'STATE': tokens.Keyword,
'STATEMENT': tokens.Keyword,
'STATIC': tokens.Keyword,
'STATISTICS': tokens.Keyword,
'STDIN': tokens.Keyword,
'STDOUT': tokens.Keyword,
'STORAGE': tokens.Keyword,
'STRICT': tokens.Keyword,
'STRUCTURE': tokens.Keyword,
'STYPE': tokens.Keyword,
'SUBCLASS_ORIGIN': tokens.Keyword,
'SUBLIST': tokens.Keyword,
'SUBSTRING': tokens.Keyword,
'SUCCESSFUL': tokens.Keyword,
'SUM': tokens.Keyword,
'SYMMETRIC': tokens.Keyword,
'SYNONYM': tokens.Keyword,
'SYSID': tokens.Keyword,
'SYSTEM': tokens.Keyword,
'SYSTEM_USER': tokens.Keyword,
'TABLE': tokens.Keyword,
'TABLE_NAME': tokens.Keyword,
'TEMP': tokens.Keyword,
'TEMPLATE': tokens.Keyword,
'TEMPORARY': tokens.Keyword,
'TERMINATE': tokens.Keyword,
'THAN': tokens.Keyword,
'TIMESTAMP': tokens.Keyword,
'TIMEZONE_HOUR': tokens.Keyword,
'TIMEZONE_MINUTE': tokens.Keyword,
'TO': tokens.Keyword,
'TOAST': tokens.Keyword,
'TRAILING': tokens.Keyword,
'TRANSATION': tokens.Keyword,
'TRANSACTIONS_COMMITTED': tokens.Keyword,
'TRANSACTIONS_ROLLED_BACK': tokens.Keyword,
'TRANSATION_ACTIVE': tokens.Keyword,
'TRANSFORM': tokens.Keyword,
'TRANSFORMS': tokens.Keyword,
'TRANSLATE': tokens.Keyword,
'TRANSLATION': tokens.Keyword,
'TREAT': tokens.Keyword,
'TRIGGER': tokens.Keyword,
'TRIGGER_CATALOG': tokens.Keyword,
'TRIGGER_NAME': tokens.Keyword,
'TRIGGER_SCHEMA': tokens.Keyword,
'TRIM': tokens.Keyword,
'TRUE': tokens.Keyword,
'TRUSTED': tokens.Keyword,
'TYPE': tokens.Keyword,
'UID': tokens.Keyword,
'UNCOMMITTED': tokens.Keyword,
'UNDER': tokens.Keyword,
'UNENCRYPTED': tokens.Keyword,
'UNION': tokens.Keyword,
'UNIQUE': tokens.Keyword,
'UNKNOWN': tokens.Keyword,
'UNLISTEN': tokens.Keyword,
'UNNAMED': tokens.Keyword,
'UNNEST': tokens.Keyword,
'UNTIL': tokens.Keyword,
'UPPER': tokens.Keyword,
'USAGE': tokens.Keyword,
'USE': tokens.Keyword,
'USER': tokens.Keyword,
'USER_DEFINED_TYPE_CATALOG': tokens.Keyword,
'USER_DEFINED_TYPE_NAME': tokens.Keyword,
'USER_DEFINED_TYPE_SCHEMA': tokens.Keyword,
'USING': tokens.Keyword,
'VACUUM': tokens.Keyword,
'VALID': tokens.Keyword,
'VALIDATE': tokens.Keyword,
'VALIDATOR': tokens.Keyword,
'VALUES': tokens.Keyword,
'VARIABLE': tokens.Keyword,
'VERBOSE': tokens.Keyword,
'VERSION': tokens.Keyword,
'VIEW': tokens.Keyword,
'VOLATILE': tokens.Keyword,
'WEEK': tokens.Keyword,
'WHENEVER': tokens.Keyword,
'WITH': tokens.Keyword.CTE,
'WITHOUT': tokens.Keyword,
'WORK': tokens.Keyword,
'WRITE': tokens.Keyword,
'YEAR': tokens.Keyword,
'ZONE': tokens.Keyword,
# Name.Builtin
'ARRAY': tokens.Name.Builtin,
'BIGINT': tokens.Name.Builtin,
'BINARY': tokens.Name.Builtin,
'BIT': tokens.Name.Builtin,
'BLOB': tokens.Name.Builtin,
'BOOLEAN': tokens.Name.Builtin,
'CHAR': tokens.Name.Builtin,
'CHARACTER': tokens.Name.Builtin,
'DATE': tokens.Name.Builtin,
'DEC': tokens.Name.Builtin,
'DECIMAL': tokens.Name.Builtin,
'FILE_TYPE': tokens.Name.Builtin,
'FLOAT': tokens.Name.Builtin,
'INT': tokens.Name.Builtin,
'INT8': tokens.Name.Builtin,
'INTEGER': tokens.Name.Builtin,
'INTERVAL': tokens.Name.Builtin,
'LONG': tokens.Name.Builtin,
'NATURALN': tokens.Name.Builtin,
'NVARCHAR': tokens.Name.Builtin,
'NUMBER': tokens.Name.Builtin,
'NUMERIC': tokens.Name.Builtin,
'PLS_INTEGER': tokens.Name.Builtin,
'POSITIVE': tokens.Name.Builtin,
'POSITIVEN': tokens.Name.Builtin,
'REAL': tokens.Name.Builtin,
'ROWID': tokens.Name.Builtin,
'ROWLABEL': tokens.Name.Builtin,
'ROWNUM': tokens.Name.Builtin,
'SERIAL': tokens.Name.Builtin,
'SERIAL8': tokens.Name.Builtin,
'SIGNED': tokens.Name.Builtin,
'SIGNTYPE': tokens.Name.Builtin,
'SIMPLE_DOUBLE': tokens.Name.Builtin,
'SIMPLE_FLOAT': tokens.Name.Builtin,
'SIMPLE_INTEGER': tokens.Name.Builtin,
'SMALLINT': tokens.Name.Builtin,
'SYS_REFCURSOR': tokens.Name.Builtin,
'SYSDATE': tokens.Name,
'TEXT': tokens.Name.Builtin,
'TINYINT': tokens.Name.Builtin,
'UNSIGNED': tokens.Name.Builtin,
'UROWID': tokens.Name.Builtin,
'UTL_FILE': tokens.Name.Builtin,
'VARCHAR': tokens.Name.Builtin,
'VARCHAR2': tokens.Name.Builtin,
'VARYING': tokens.Name.Builtin,
}
KEYWORDS_COMMON = {
'SELECT': tokens.Keyword.DML,
'INSERT': tokens.Keyword.DML,
'DELETE': tokens.Keyword.DML,
'UPDATE': tokens.Keyword.DML,
'UPSERT': tokens.Keyword.DML,
'REPLACE': tokens.Keyword.DML,
'MERGE': tokens.Keyword.DML,
'DROP': tokens.Keyword.DDL,
'CREATE': tokens.Keyword.DDL,
'ALTER': tokens.Keyword.DDL,
'TRUNCATE': tokens.Keyword.DDL,
'GRANT': tokens.Keyword.DCL,
'REVOKE': tokens.Keyword.DCL,
'WHERE': tokens.Keyword,
'FROM': tokens.Keyword,
'INNER': tokens.Keyword,
'JOIN': tokens.Keyword,
'STRAIGHT_JOIN': tokens.Keyword,
'AND': tokens.Keyword,
'OR': tokens.Keyword,
'LIKE': tokens.Keyword,
'ON': tokens.Keyword,
'IN': tokens.Keyword,
'SET': tokens.Keyword,
'BY': tokens.Keyword,
'GROUP': tokens.Keyword,
'ORDER': tokens.Keyword,
'LEFT': tokens.Keyword,
'OUTER': tokens.Keyword,
'FULL': tokens.Keyword,
'IF': tokens.Keyword,
'END': tokens.Keyword,
'THEN': tokens.Keyword,
'LOOP': tokens.Keyword,
'AS': tokens.Keyword,
'ELSE': tokens.Keyword,
'FOR': tokens.Keyword,
'WHILE': tokens.Keyword,
'CASE': tokens.Keyword,
'WHEN': tokens.Keyword,
'MIN': tokens.Keyword,
'MAX': tokens.Keyword,
'DISTINCT': tokens.Keyword,
}
KEYWORDS_ORACLE = {
'ARCHIVE': tokens.Keyword,
'ARCHIVELOG': tokens.Keyword,
'BACKUP': tokens.Keyword,
'BECOME': tokens.Keyword,
'BLOCK': tokens.Keyword,
'BODY': tokens.Keyword,
'CANCEL': tokens.Keyword,
'CHANGE': tokens.Keyword,
'COMPILE': tokens.Keyword,
'CONTENTS': tokens.Keyword,
'CONTROLFILE': tokens.Keyword,
'DATAFILE': tokens.Keyword,
'DBA': tokens.Keyword,
'DISMOUNT': tokens.Keyword,
'DOUBLE': tokens.Keyword,
'DUMP': tokens.Keyword,
'ELSIF': tokens.Keyword,
'EVENTS': tokens.Keyword,
'EXCEPTIONS': tokens.Keyword,
'EXPLAIN': tokens.Keyword,
'EXTENT': tokens.Keyword,
'EXTERNALLY': tokens.Keyword,
'FLUSH': tokens.Keyword,
'FREELIST': tokens.Keyword,
'FREELISTS': tokens.Keyword,
# groups seems too common as table name
# 'GROUPS': tokens.Keyword,
'INDICATOR': tokens.Keyword,
'INITRANS': tokens.Keyword,
'INSTANCE': tokens.Keyword,
'LAYER': tokens.Keyword,
'LINK': tokens.Keyword,
'LISTS': tokens.Keyword,
'LOGFILE': tokens.Keyword,
'MANAGE': tokens.Keyword,
'MANUAL': tokens.Keyword,
'MAXDATAFILES': tokens.Keyword,
'MAXINSTANCES': tokens.Keyword,
'MAXLOGFILES': tokens.Keyword,
'MAXLOGHISTORY': tokens.Keyword,
'MAXLOGMEMBERS': tokens.Keyword,
'MAXTRANS': tokens.Keyword,
'MINEXTENTS': tokens.Keyword,
'MODULE': tokens.Keyword,
'MOUNT': tokens.Keyword,
'NOARCHIVELOG': tokens.Keyword,
'NOCACHE': tokens.Keyword,
'NOCYCLE': tokens.Keyword,
'NOMAXVALUE': tokens.Keyword,
'NOMINVALUE': tokens.Keyword,
'NOORDER': tokens.Keyword,
'NORESETLOGS': tokens.Keyword,
'NORMAL': tokens.Keyword,
'NOSORT': tokens.Keyword,
'OPTIMAL': tokens.Keyword,
'OWN': tokens.Keyword,
'PACKAGE': tokens.Keyword,
'PARALLEL': tokens.Keyword,
'PCTINCREASE': tokens.Keyword,
'PCTUSED': tokens.Keyword,
'PLAN': tokens.Keyword,
'PRIVATE': tokens.Keyword,
'PROFILE': tokens.Keyword,
'QUOTA': tokens.Keyword,
'RECOVER': tokens.Keyword,
'RESETLOGS': tokens.Keyword,
'RESTRICTED': tokens.Keyword,
'REUSE': tokens.Keyword,
'ROLES': tokens.Keyword,
'SAVEPOINT': tokens.Keyword,
'SCN': tokens.Keyword,
'SECTION': tokens.Keyword,
'SEGMENT': tokens.Keyword,
'SHARED': tokens.Keyword,
'SNAPSHOT': tokens.Keyword,
'SORT': tokens.Keyword,
'STATEMENT_ID': tokens.Keyword,
'STOP': tokens.Keyword,
'SWITCH': tokens.Keyword,
'TABLES': tokens.Keyword,
'TABLESPACE': tokens.Keyword,
'THREAD': tokens.Keyword,
'TIME': tokens.Keyword,
'TRACING': tokens.Keyword,
'TRANSACTION': tokens.Keyword,
'TRIGGERS': tokens.Keyword,
'UNLIMITED': tokens.Keyword,
'UNLOCK': tokens.Keyword,
}
# MySQL
KEYWORDS_MYSQL = {
'ROW': tokens.Keyword,
}
# PostgreSQL Syntax
KEYWORDS_PLPGSQL = {
'CONFLICT': tokens.Keyword,
'WINDOW': tokens.Keyword,
'PARTITION': tokens.Keyword,
'ATTACH': tokens.Keyword,
'DETACH': tokens.Keyword,
'OVER': tokens.Keyword,
'PERFORM': tokens.Keyword,
'NOTICE': tokens.Keyword,
'PLPGSQL': tokens.Keyword,
'INHERIT': tokens.Keyword,
'INDEXES': tokens.Keyword,
'ON_ERROR_STOP': tokens.Keyword,
'EXTENSION': tokens.Keyword,
'BYTEA': tokens.Keyword,
'BIGSERIAL': tokens.Keyword,
'BIT VARYING': tokens.Keyword,
'BOX': tokens.Keyword,
'CHARACTER': tokens.Keyword,
'CHARACTER VARYING': tokens.Keyword,
'CIDR': tokens.Keyword,
'CIRCLE': tokens.Keyword,
'DOUBLE PRECISION': tokens.Keyword,
'INET': tokens.Keyword,
'JSON': tokens.Keyword,
'JSONB': tokens.Keyword,
'LINE': tokens.Keyword,
'LSEG': tokens.Keyword,
'MACADDR': tokens.Keyword,
'MONEY': tokens.Keyword,
'PATH': tokens.Keyword,
'PG_LSN': tokens.Keyword,
'POINT': tokens.Keyword,
'POLYGON': tokens.Keyword,
'SMALLSERIAL': tokens.Keyword,
'TSQUERY': tokens.Keyword,
'TSVECTOR': tokens.Keyword,
'TXID_SNAPSHOT': tokens.Keyword,
'UUID': tokens.Keyword,
'XML': tokens.Keyword,
'FOR': tokens.Keyword,
'IN': tokens.Keyword,
'LOOP': tokens.Keyword,
}
# Hive Syntax
KEYWORDS_HQL = {
'EXPLODE': tokens.Keyword,
'DIRECTORY': tokens.Keyword,
'DISTRIBUTE': tokens.Keyword,
'INCLUDE': tokens.Keyword,
'LOCATE': tokens.Keyword,
'OVERWRITE': tokens.Keyword,
'POSEXPLODE': tokens.Keyword,
'ARRAY_CONTAINS': tokens.Keyword,
'CMP': tokens.Keyword,
'COLLECT_LIST': tokens.Keyword,
'CONCAT': tokens.Keyword,
'CONDITION': tokens.Keyword,
'DATE_ADD': tokens.Keyword,
'DATE_SUB': tokens.Keyword,
'DECODE': tokens.Keyword,
'DBMS_OUTPUT': tokens.Keyword,
'ELEMENTS': tokens.Keyword,
'EXCHANGE': tokens.Keyword,
'EXTENDED': tokens.Keyword,
'FLOOR': tokens.Keyword,
'FOLLOWING': tokens.Keyword,
'FROM_UNIXTIME': tokens.Keyword,
'FTP': tokens.Keyword,
'HOUR': tokens.Keyword,
'INLINE': tokens.Keyword,
'INSTR': tokens.Keyword,
'LEN': tokens.Keyword,
'MAP': tokens.Name.Builtin,
'MAXELEMENT': tokens.Keyword,
'MAXINDEX': tokens.Keyword,
'MAX_PART_DATE': tokens.Keyword,
'MAX_PART_INT': tokens.Keyword,
'MAX_PART_STRING': tokens.Keyword,
'MINELEMENT': tokens.Keyword,
'MININDEX': tokens.Keyword,
'MIN_PART_DATE': tokens.Keyword,
'MIN_PART_INT': tokens.Keyword,
'MIN_PART_STRING': tokens.Keyword,
'NOW': tokens.Keyword,
'NVL': tokens.Keyword,
'NVL2': tokens.Keyword,
'PARSE_URL_TUPLE': tokens.Keyword,
'PART_LOC': tokens.Keyword,
'PART_COUNT': tokens.Keyword,
'PART_COUNT_BY': tokens.Keyword,
'PRINT': tokens.Keyword,
'PUT_LINE': tokens.Keyword,
'RANGE': tokens.Keyword,
'REDUCE': tokens.Keyword,
'REGEXP_REPLACE': tokens.Keyword,
'RESIGNAL': tokens.Keyword,
'RTRIM': tokens.Keyword,
'SIGN': tokens.Keyword,
'SIGNAL': tokens.Keyword,
'SIN': tokens.Keyword,
'SPLIT': tokens.Keyword,
'SQRT': tokens.Keyword,
'STACK': tokens.Keyword,
'STR': tokens.Keyword,
'STRING': tokens.Name.Builtin,
'STRUCT': tokens.Name.Builtin,
'SUBSTR': tokens.Keyword,
'SUMMARY': tokens.Keyword,
'TBLPROPERTIES': tokens.Keyword,
'TIMESTAMP': tokens.Name.Builtin,
'TIMESTAMP_ISO': tokens.Keyword,
'TO_CHAR': tokens.Keyword,
'TO_DATE': tokens.Keyword,
'TO_TIMESTAMP': tokens.Keyword,
'TRUNC': tokens.Keyword,
'UNBOUNDED': tokens.Keyword,
'UNIQUEJOIN': tokens.Keyword,
'UNIX_TIMESTAMP': tokens.Keyword,
'UTC_TIMESTAMP': tokens.Keyword,
'VIEWS': tokens.Keyword,
'EXIT': tokens.Keyword,
'BREAK': tokens.Keyword,
'LEAVE': tokens.Keyword,
}
KEYWORDS_MSACCESS = {
'DISTINCTROW': tokens.Keyword,
}
KEYWORDS_SNOWFLAKE = {
'ACCOUNT': tokens.Keyword,
'GSCLUSTER': tokens.Keyword,
'ISSUE': tokens.Keyword,
'ORGANIZATION': tokens.Keyword,
'PIVOT': tokens.Keyword,
'QUALIFY': tokens.Keyword,
'REGEXP': tokens.Keyword,
'RLIKE': tokens.Keyword,
'SAMPLE': tokens.Keyword,
'TRY_CAST': tokens.Keyword,
'UNPIVOT': tokens.Keyword,
'VARIANT': tokens.Name.Builtin,
}
KEYWORDS_BIGQUERY = {
'ASSERT_ROWS_MODIFIED': tokens.Keyword,
'DEFINE': tokens.Keyword,
'ENUM': tokens.Keyword,
'HASH': tokens.Keyword,
'LOOKUP': tokens.Keyword,
'PRECEDING': tokens.Keyword,
'PROTO': tokens.Keyword,
'RESPECT': tokens.Keyword,
'TABLESAMPLE': tokens.Keyword,
'BIGNUMERIC': tokens.Name.Builtin,
}