statement_splitter.py 7.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196
  1. #
  2. # Copyright (C) 2009-2020 the sqlparse authors and contributors
  3. # <see AUTHORS file>
  4. #
  5. # This module is part of python-sqlparse and is released under
  6. # the BSD License: https://opensource.org/licenses/BSD-3-Clause
  7. from sqlparse import sql
  8. from sqlparse import tokens as T
  9. class StatementSplitter:
  10. """Filter that split stream at individual statements"""
  11. def __init__(self):
  12. self._reset()
  13. def _reset(self):
  14. """Set the filter attributes to its default values"""
  15. self._block_stack = []
  16. self._parenthesis_level = 0
  17. self._unconfirmed_start = None
  18. self._is_create = False
  19. self._seen_begin = False
  20. self.consume_ws = False
  21. self.tokens = []
  22. self.level = 0
  23. def _handle_nested_block(self, unified):
  24. """Check for nested loop or control structures inside a block"""
  25. if unified == 'FOR':
  26. self._unconfirmed_start = 'FOR'
  27. return 0
  28. if unified == 'WHILE':
  29. self._unconfirmed_start = 'WHILE'
  30. return 0
  31. if unified in ('LOOP', 'DO'):
  32. if self._unconfirmed_start in ('FOR', 'WHILE'):
  33. self._block_stack.append(self._unconfirmed_start)
  34. self._unconfirmed_start = None
  35. return 1
  36. if unified == 'LOOP':
  37. self._block_stack.append('LOOP')
  38. return 1
  39. if unified in ('IF', 'CASE'):
  40. self._block_stack.append(unified)
  41. return 1
  42. return None
  43. def _handle_closing_keyword(self, unified):
  44. """Handle closing keywords for blocks"""
  45. if unified == 'END IF':
  46. if self._block_stack and self._block_stack[-1] == 'IF':
  47. self._block_stack.pop()
  48. return -1
  49. elif unified == 'END FOR':
  50. if self._block_stack and self._block_stack[-1] == 'FOR':
  51. self._block_stack.pop()
  52. return -1
  53. elif unified == 'END WHILE':
  54. if self._block_stack and self._block_stack[-1] == 'WHILE':
  55. self._block_stack.pop()
  56. return -1
  57. elif unified == 'END LOOP':
  58. if (self._block_stack and
  59. self._block_stack[-1] in ('LOOP', 'FOR', 'WHILE')):
  60. self._block_stack.pop()
  61. return -1
  62. elif unified == 'END CASE':
  63. if self._block_stack and self._block_stack[-1] == 'CASE':
  64. self._block_stack.pop()
  65. return -1
  66. elif unified == 'END':
  67. if self._block_stack:
  68. self._block_stack.pop()
  69. return -1
  70. return 0
  71. def _change_splitlevel(self, ttype, value):
  72. """Get the new split level (increase, decrease or remain equal)"""
  73. # Semicolon resets unconfirmed loop starters
  74. # and handles standalone BEGIN;
  75. if ttype is T.Punctuation and value == ';':
  76. self._unconfirmed_start = None
  77. if self._seen_begin:
  78. self._seen_begin = False
  79. if self._block_stack and self._block_stack[-1] == 'BEGIN':
  80. self._block_stack.pop()
  81. return -1
  82. return 0
  83. # parenthesis increase/decrease a level
  84. if ttype is T.Punctuation and value == '(':
  85. self._parenthesis_level += 1
  86. return 1
  87. elif ttype is T.Punctuation and value == ')':
  88. self._parenthesis_level = max(0, self._parenthesis_level - 1)
  89. return -1
  90. elif ttype not in T.Keyword: # if normal token return
  91. return 0
  92. # Everything after here is ttype = T.Keyword
  93. unified = value.upper()
  94. # DDL Create though can contain more words such as "or replace"
  95. if ttype is T.Keyword.DDL and unified.startswith('CREATE'):
  96. self._is_create = True
  97. return 0
  98. # Handle DECLARE block start (only for CREATE statements)
  99. if unified == 'DECLARE' and self._is_create and not self._block_stack:
  100. self._block_stack.append('DECLARE')
  101. return 1
  102. # Handle BEGIN block start
  103. if unified == 'BEGIN':
  104. self._seen_begin = True
  105. # Transition DECLARE to BEGIN if present
  106. if self._block_stack and self._block_stack[-1] == 'DECLARE':
  107. self._block_stack.pop()
  108. self._block_stack.append('BEGIN')
  109. return 0
  110. else:
  111. self._block_stack.append('BEGIN')
  112. return 1
  113. # Issue826: If we see a transaction keyword after BEGIN,
  114. # it's a transaction statement, not a block.
  115. if self._seen_begin and \
  116. (ttype is T.Keyword or ttype is T.Name) and \
  117. unified in ('TRANSACTION', 'WORK', 'TRAN',
  118. 'DISTRIBUTED', 'DEFERRED',
  119. 'IMMEDIATE', 'EXCLUSIVE'):
  120. self._seen_begin = False
  121. if self._block_stack and self._block_stack[-1] == 'BEGIN':
  122. self._block_stack.pop()
  123. return -1
  124. return 0
  125. # Inside a block, check for nested loop or control structures
  126. if 'BEGIN' in self._block_stack:
  127. res = self._handle_nested_block(unified)
  128. if res is not None:
  129. return res
  130. # Handle closing keywords
  131. return self._handle_closing_keyword(unified)
  132. def process(self, stream):
  133. """Process the stream"""
  134. EOS_TTYPE = T.Whitespace, T.Comment.Single
  135. # Run over all stream tokens
  136. for ttype, value in stream:
  137. # Yield token if we finished a statement and there's no whitespaces
  138. # It will count newline token as a non whitespace. In this context
  139. # whitespace ignores newlines.
  140. # why don't multi line comments also count?
  141. if self.consume_ws and ttype not in EOS_TTYPE:
  142. yield sql.Statement(self.tokens)
  143. # Reset filter and prepare to process next statement
  144. self._reset()
  145. # Change current split level (increase, decrease or remain equal)
  146. self.level += self._change_splitlevel(ttype, value)
  147. # Append the token to the current statement
  148. self.tokens.append(sql.Token(ttype, value))
  149. # Check if we get the end of a statement
  150. # Issue762: Allow GO (or "GO 2") as statement splitter.
  151. # When implementing a language toggle, it's not only to add
  152. # keywords it's also to change some rules, like this splitting
  153. # rule.
  154. # Issue809: Ignore semicolons inside BEGIN...END blocks, but handle
  155. # standalone BEGIN; as a transaction statement
  156. if ttype is T.Punctuation and value == ';':
  157. self._seen_begin = False
  158. # Split on semicolon if not inside a BEGIN...END block
  159. if self.level <= 0 and 'BEGIN' not in self._block_stack:
  160. self.consume_ws = True
  161. elif ttype is T.Keyword and value.split()[0] == 'GO':
  162. self.consume_ws = True
  163. elif (ttype not in (T.Whitespace, T.Newline, T.Comment.Single,
  164. T.Comment.Multiline)
  165. and not (ttype is T.Keyword and value.upper() == 'BEGIN')):
  166. # Reset _seen_begin if we see a non-whitespace, non-comment
  167. # token but not for BEGIN itself (which just set the flag)
  168. self._seen_begin = False
  169. # Yield pending statement (if any)
  170. if self.tokens and not all(t.is_whitespace for t in self.tokens):
  171. yield sql.Statement(self.tokens)