# imba$inlineHelpers=1
# imba$v2=0

var T = require './token'
var Token = T.Token

import INVERSES from './constants'
import Compilation from './compilation'

var K = 0

var ERR = require './errors'
var helpers = require './helpers'

# Constants
# ---------

# Keywords that Imba shares in common with JavaScript.
var JS_KEYWORDS = [
	'true', 'false', 'null', 'this'
	'delete', 'typeof', 'in', 'instanceof'
	'throw', 'break', 'continue', 'debugger'
	'if', 'else', 'switch', 'for', 'while', 'do', 'try', 'catch', 'finally'
	'class', 'extends', 'super', 'return', 'yield'
]

var TSC_CARET_BEFORE = {
	',': 1
	'\n': 1
	')': 1
	']': 1
	'}': 1
	'>': 1
	' ': 1
}

var TYPE_GENERICS_AFTER = /\w|\]|\)$/

# new can be used as a keyword in imba, since object initing is done through
# MyObject.new. new is a very useful varname.

# We want to treat return like any regular call for now
# Must be careful to throw the exceptions in AST, since the parser
# wont

# Imba-only keywords. var should move to JS_Keywords
# some words (like tokid) should be context-specific
var IMBA_KEYWORDS = [
	'undefined', 'then', 'unless', 'until', 'loop', 'of', 'by',
	'when','def','tag','do','elif','begin','var','let','const','await','import','module','export','static','extend','yield'
]

var IMBA_CONTEXTUAL_KEYWORDS = ['extend','local','global','prop']

var CONTEXTUAL_KEYWORDS = {
	LET: {'global': 1, 'declare': 1, static: 1}
	CONST: {'global': 1, 'declare': 1, static: 1}
	VAR: {'global': 1, 'declare': 1, static: 1}
	CLASS: {'global': 1, 'declare': 1, static: 1, extend: 1, strict: 1, abstract: 1, mixin: 1}
	MIXIN: {'global': 1, 'declare': 1, extend: 1, strict: 1, abstract: 1}
	INTERFACE: {'global': 1, 'declare': 1, extend: 1, strict: 1, mixin: 1}
	TAG: {'global': 1, 'declare': 1, extend: 1, strict: 1, abstract: 1}
	DEF: {'global': 1, 'declare': 1, protected: 1}
	PROP: {'static': 1}
	ATTR: {'static': 1}
	CSS: {'global': 1, export: 1}
}

var MEMBER_KEYWORDS = {
	DEF: 1
	PROP: 1
	ATTR: 1
}

# FixedArray for performance
# var ALL_KEYWORDS = JS_KEYWORDS.concat(IMBA_KEYWORDS)
export var ALL_KEYWORDS = [
	'true', 'false', 'null', 'this','self'
	'delete', 'typeof', 'in', 'instanceof',
	'throw', 'break', 'continue', 'debugger',
	'if', 'else', 'switch', 'for', 'while', 'do', 'try', 'catch', 'finally',
	'class', 'extends', 'super', 'return',
	'undefined', 'then', 'unless', 'until', 'loop', 'of', 'by',
	'when','def','tag','do','elif','begin','var','let','const','await','import',
	'and','or','is','isnt','not','yes','no','isa','case','nil','module','export','static','extend'
	'yield'
]

# The list of keywords that are reserved by JavaScript, but not used, or are
# used by Imba internally. We throw an error when these are encountered,
# to avoid having a JavaScript error at runtime.  # 'var', 'let', - not inside here
var RESERVED = ['case', 'default', 'function', 'void', 'with', 'const', 'enum', 'native']
var STRICT_RESERVED = ['case','function','void','const']

# The superset of both JavaScript keywords and reserved words, none of which may
# be used as identifiers or properties.
var JS_FORBIDDEN = JS_KEYWORDS.concat RESERVED

var METHOD_IDENTIFIER = /// ^
	(
		(([\x23]?[\$A-Za-z_\x7f-\uffff][$\-\w\x7f-\uffff]*)([\=]?)) |
		(<=>|\|(?![\|=]))
	)
///
# removed ~=|~| |&(?![&=])

# Token matching regexes.
# added hyphens to identifiers now - to test
var IDENTIFIER = /// ^
	(
		(\$|##|#|@@|@|\%)[\$\wA-Za-z_\-\x7f-\uffff][$\w\x7f-\uffff]* (\-[$\w\x7f-\uffff]+)* [\?]? |
		[$A-Za-z_][$\w\x7f-\uffff]* (\-[$\w\x7f-\uffff]+)* [\?]?
	)
	( [^\n\S]* : )?  # Is this a property name?
///

var IMPORTS = /// ^
	import\s+(\{?[^\"\'\}]+\}?)(?=\s+from\s+)
///

var OBJECT_KEY = /// ^
	( (\$?)[$A-Za-z_\x7f-\uffff\-][$\w\x7f-\uffff\-]*)
	( [^\n\S\s]* : (?![\*\=:$A-Za-z\_\x7f-\uffff]) )  # Is this a property name?
///

var TAG = /// ^
	(\<)(?=[A-Za-z\#\.\%\$\[\{\@\>\(])
///

var TAG_TYPE = /^(\w[\w\d]*:)?(\w[\w\d]*)(-[\w\d]+)*/
var TAG_ID = /^#((\w[\w\d]*)(-[\w\d]+)*)/
var TAG_PART = /^[\:\.\#]?([A-Za-z\_][\w\-]*)(\:[A-Za-z\_][\w\-]*)?/
var TAG_ATTR = /^([\.\:]?[\w\_]+([\-\:\.][\w]+)*)(\s)*\=(?!\>)/

var SELECTOR = /^([%\$]{1,2})([\(])/
var SELECTOR_PART = /^(\#|\.|:|::)?([\w]+(\-[\w]+)*)/
var SELECTOR_COMBINATOR = /^ (\+|\>|\~)*\s*(?=[\w\.\#\:\{\*\[])/

var SELECTOR_PSEUDO_CLASS = /^(::?)([\w]+(\-[\w]+)*)/
var SELECTOR_ATTR_OP = /^(\$=|\~=|\^=|\*=|\|=|=|\!=)/
var SELECTOR_ATTR = /^\[([\w\_\-]+)(\$=|\~=|\^=|\*=|\|=|=|\!=)/

var SYMBOL = ///^
	\:((([\*\@$\w\x7f-\uffff]+)+([\-\\\:][\w\x7f-\uffff]+)*)|==|\<=\>)
///

var STYLE_HEX = ///^
	\#[\w\-]+
///

var STYLE_NUMERIC = ///^
	(\-?\d*\.?\d+) ([A-Za-z]+|\%)?(?![\d\w])
///

var STYLE_IDENTIFIER = ///^
	[\w\-\$]*\w[\w\-\$]*
///

var STYLE_URL = ///^
	url\(([^\)]*)\)
///

var STYLE_PROPERTY = ///^
	(\^?[\w\-\$\@\.\!\#\^]+)(?=\:([^\:]|$)|\s*\=)
///

var STYLE_MODIFIERS = ///^
	(\@?[\w\-\$]*\w[\w\-\$]*)
	([\.\@][\w\-\$]*\w[\w\-\$]*)*
	(\@[\w\-\$]*\w[\w\-\$]*)*
	(?=\:)
///

var NUMBER = ///
	^ 0x[\da-f_]+ |                              # hex
	^ 0b[01_]+ |                              # binary
	^ 0o[\d_]+ |                              # binary
	^ [\-]?(?:\d[_\d]*)\.?\d[_\d]* (?:e[+-]?\d+)? | # decimal2
	^ [\-]?\d*\.?\d+ (?:e[+-]?\d+)? # decimal

///i

var HEREDOC = /// ^ ("""|''') ([\s\S]*?) (?:\n[^\n\S]*)? \1 ///

var OPERATOR = /// ^ (
	?: [-=]=>             # function - what
	 | !& (?=[\s\n])
	 | [&|~^]?=\? # reassign
	 | [&|~^]= # bit assign
	 | \?\?= # nullish coallescing assignment
	 | ===
	 | ---
	 | ->
	 | =>
	 | />
	 | !==
	 | \*\*=?            # exponentation
	 | [-+*/%<>&|^!?=]=  # compound assign / compare
	 | =<
	 | >>>=?             # zero-fill right shift
	 | ([-+:])\1         # doubles
	 | ([&|<>])\2=?      # logic / shift
	 | \?\.              # soak access
	 | \?\?              # soak symbol
	 | \.{2,3}           # range or splat
	 | \*(?=[a-zA-Z\_])   # splat --
) ///

# FIXME splat should only be allowed when the previous thing is spaced or inside call?

var WHITESPACE = /^[^\n\S]+/

var COMMENT    = /^###([^#][\s\S]*?)(?:###[^\n\S]*|(?:###)?$)/
var JS_COMMENT    = /^\/\*([\s\S]*?)\*\//
# var INLINE_COMMENT = /^(\s*)((#[ \t\!]||\/\/)(.*)|#[ \t]?(?=\n|$))+/
var INLINE_COMMENT = /^(\s*)((#[ \t\!]|\/\/(?!\/))(.*)|#[ \t]?(?=\n|$))+/

var CODE       = /^[-=]=>/

var MULTI_DENT = /^(?:\n[^\n\S]*)+/

var SIMPLESTR  = /^'[^\\']*(?:\\.[^\\']*)*'/

var JSTOKEN    = /^`[^\\`]*(?:\\.[^\\`]*)*`/

# Regex-matching-regexes.
var REGEX = /// ^
	(/ (?! [\s=] )   # disallow leading whitespace or equals signs
	[^ [ / \n \\ ]*  # every other thing
	(?:
		(?: \\[\s\S]   # anything escaped
			| \[         # character class
					 [^ \] \n \\ ]*
					 (?: \\[\s\S] [^ \] \n \\ ]* )*
				 ]
		) [^ [ / \n \\ ]*
	)*
	/) ([a-z]{0,8}) (?!\w)
///

var HEREGEX      = /// ^ /{3} ([\s\S]+?) /{3} ([a-z]{0,8}) (?!\w) ///

var HEREGEX_OMIT = /\s+(?:#.*)?/g

# Token cleaning regexes.
var MULTILINER      = /\n/g

var HEREDOC_INDENT  = /\n+([^\n\S]*)/g

var HEREDOC_ILLEGAL = /\*\//

# expensive?
var LINE_CONTINUER  = /// ^ \s* (?: , | \??\.(?![.\d]) | (?:&&|\|\||and|or)[\n\s] ) ///

var TRAILING_SPACES = /\s+$/

var ENV_FLAG = /^\$\w+\$/

var ARGVAR = /^\$\d$/

# Compound assignment tokens.
var COMPOUND_ASSIGN = [
	'-=', '+=', '/=', '*=', '%=', '||=', '&&=', '?=','??=',
	'<<=', '>>=', '>>>=', '&=', '^=', '|=', '~=', '=<', '**=',
	'=?','~=?','|=?','&=?','^=?'
]

# Unary tokens.
var UNARY = ['!', '~', 'NEW', 'TYPEOF', 'DELETE']

# Logical tokens.
var LOGIC   = ['&&', '||', '??', 'and','or']

# Bit-shifting tokens.
var SHIFT   = ['<<', '>>', '>>>']

# Comparison tokens.
var COMPARE = ['===', '!==', '==', '!=', '<', '>', '<=', '>=','===','!==','&', '|', '^','!&']

# Mathematical tokens.
var MATH = ['*', '/', '%', '∪', '∩', '√']

# Relational tokens that are negatable with `not` prefix.
var RELATION = ['IN', 'OF', 'INSTANCEOF','ISA']

# Boolean tokens.
var BOOL = ['TRUE', 'FALSE', 'NULL', 'UNDEFINED']

# Our list is shorter, due to sans-parentheses method calls.
var NOT_REGEX = ['NUMBER', 'REGEX', 'BOOL', 'TRUE', 'FALSE', '++', '--', ']']

# If the previous token is not spaced, there are more preceding tokens that
# force a division parse:
var NOT_SPACED_REGEX = ['NUMBER', 'REGEX', 'BOOL', 'TRUE', 'FALSE', '++', '--', ']',')', '}', 'THIS', 'SELF' , 'IDENTIFIER', 'STRING']

# Tokens which could legitimately be invoked or indexed. An opening
# parentheses or bracket following these tokens will be recorded as the start
# of a function invocation or indexing operation.
# really?!

var UNFINISHED = ['\\','.','UNARY', 'MATH', 'EXP', '+', '-', 'SHIFT', 'RELATION', 'COMPARE', 'THROW', 'EXTENDS']

# } should not be callable anymore!!! '}', '::',
var CALLABLE  = ['IDENTIFIER','SYMBOLID', 'STRING', 'REGEX', ')', ']','INDEX_END', 'THIS', 'SUPER', 'TAG_END', 'IVAR', 'SELF','NEW','ARGVAR','SYMBOL','RETURN','INDEX_END','CALL_END','DECORATOR','@','GENERICS']

# optimize for FixedArray
var INDEXABLE = [
	'IDENTIFIER', 'SYMBOLID', 'STRING', 'REGEX', ')', ']', 'THIS', 'SUPER', 'TAG_END', 'IVAR', 'SELF','NEW','ARGVAR','SYMBOL','RETURN','BANG',
	'NUMBER', 'BOOL', 'TAG_SELECTOR', 'ARGUMENTS','}','TAG_TYPE','TAG_REF','INDEX_END','CALL_END','DO_VALUE'
]

var NOT_KEY_AFTER = ['.','?','?.','UNARY','??','+','-','*']

var GLOBAL_IDENTIFIERS = ['global','exports']

# Tokens that, when immediately preceding a `WHEN`, indicate that the `WHEN`
# occurs at the start of a line. We disambiguate these from trailing whens to
# avoid an ambiguity in the grammar.
var LINE_BREAK = ['INDENT', 'OUTDENT', 'TERMINATOR']

export class LexerError < SyntaxError

	def initialize message, file, line
		self:message = message
		self:file = file
		self:line = line
		return self

var last = do |array, back = 0|
	array[array:length - back - 1]

var count = do |str, substr|
	return str.split(substr):length - 1

var repeatString = do |str,times|
	var res = ''
	while times > 0
		if times % 2 == 1
			res += str
		str += str
		times >>= 1
	return res

var tT  = T:typ
var tV  = T:val
var tTs = T:setTyp
var tVs = T:setVal

# The Lexer class reads a stream of Imba and divvies it up into tokidged
# tokens. Some potential ambiguity in the grammar has been avoided by
# pushing some extra smarts into the Lexer.

# Based on the original lexer.coffee from CoffeeScript
export class Lexer

	def initialize
		reset
		self

	def reset
		@code    = null
		@chunk   = null           # The remainder of the source code.
		@opts    = null
		@state = {}

		@indent  = 0              # The current indentation level.
		@indebt  = 0              # The over-indentation at the current level.
		@outdebt = 0              # The under-outdentation at the current level.

		@indents  = []             # The stack of all current indentation levels.
		@ends     = [] # The stack for pairing up tokens.
		@contexts = [] # suplements @ends
		@scopes   = []
		@nextScope = null # the scope to add on the next indent
		@context = null
		# should rather make it like a statemachine that moves from CLASS_DEF to CLASS_BODY etc
		# Things should compile differently when you are in a CLASS_BODY than when in a DEF_BODY++

		@indentStyle = '\t'
		@inTag = no
		@inStyle = 0

		@tokens  = []             # Stream of parsed tokens in the form `['TYPE', value, line]`.
		@seenFor = no
		@loc = 0
		@locOffset = 0

		@end     = null
		@char 	 = null
		@bridge  = null

		@last    = null
		@lastTyp = ''
		@lastVal = null
		@script  = null
		self

	def jisonBridge jison
		@bridge = {
			lex: T:lex
			setInput: do |tokens|
				this:tokens = tokens
				this:pos = 0

			upcomingInput: do ""
		}

	def tokenize code, o, script = null

		if code:length == 0
			return []

		unless o:inline
			if WHITESPACE.test(code)
				code = "\n{code}"
				return [] if code.match(/^\s*$/g)

			code = code.replace(/\r/g, '').replace /[\t ]+$/g, ''

		@last    = null
		@lastTyp = null
		@lastVal = null

		@script  = script
		@code    = code
		@opts    = o
		@locOffset = o:loc or 0
		@platform = o:platform or o:target
		@indentStyle = '\t'

		# if the very first line is indented, take this as a gutter
		if let m = code.match(/^([\ \t]*)[^\n\s\t]/)
			@state:gutter = m[1]

		if o:gutter !== undefined
			@state:gutter = o:gutter

		if @script and !o:inline
			@script:tokens = @tokens

		parse(code)
		closeIndentation unless o:inline

		

		if @ends:length
			error("missing {@ends.pop}")

		if @platform == 'tsc'
			for token in @tokens
				if token.@type == 'SYMBOLID'
					token.@type = 'IDENTIFIER'
					# token.@value = token.@value.replace(/#/g,'_$SYM$_')

		return @tokens

	def parse code
		var i = 0
		var pi = 0

		@loc = @locOffset + i

		while @chunk = code.slice(i)
			let ctx = @context
			if ctx and ctx:pop
				if ctx:pop.test(@chunk)
					popEnd

			# we should let the current context decide which methods to call
			pi = (ctx and ctx:lexer and ctx:lexer.call(self)) || (@end == 'TAG' and tagDefContextToken) || (@inTag and tagContextToken) || (@inStyle2 and lexStyleBody) || basicContext
			i += pi
			@loc = @locOffset + i

		return

	def basicContext
		return selectorToken || symbolToken || identifierToken || whitespaceToken || lineToken || commentToken || heredocToken || tagToken || stringToken || numberToken || regexToken || literalToken || 0

	def prevChars n = 1
		n == 1 ? @code[@loc - 1] : @code.slice(@loc - n,@loc)

	def moveCaret i
		@loc += i

	def context
		@ends[@ends:length - 1]

	def inContext key
		var o = @contexts[@contexts:length - 1]
		return o and o[key]

	def pushEnd val, ctx
		let prev = @context
		@ends.push(val)
		@contexts.push(@context = (ctx or {}))
		@end = val
		refreshScope

		if ctx and (ctx:closeType == 'STYLE_END' or ctx:style)
			ctx:lexer = self:lexStyleBody
			ctx:style = yes
			@inStyle++

		if prev and prev:style and val != '}'
			ctx:lexer = self:lexStyleBody
			ctx:style = yes

		if ctx and ctx:id
			ctx:start = Token.new(ctx:id + '_START',val, @last.region[1],0)
			@tokens.push(ctx:start)
		self

	def popEnd val
		var popped = @ends.pop
		@end = @ends[@ends:length - 1]

		# automatically adding a closer if this is defined
		var ctx = @context
		if ctx and ctx:start
			ctx:end = Token.new(ctx:closeType or ctx:id + '_END',popped,@last.region[1],0)
			ctx:end.@start = ctx:start
			ctx:start.@end = ctx:end
			@tokens.push(ctx:end)

		if ctx and (ctx:closeType == 'STYLE_END' or ctx:style)
			@inStyle--

		@contexts.pop
		@context = @contexts[@contexts:length - 1]

		refreshScope
		return [popped,ctx]

	def refreshScope
		var ctx0 = @ends[@ends:length - 1]
		var ctx1 = @ends[@ends:length - 2]
		@inTag = ctx0 == 'TAG_END' or (ctx1 == 'TAG_END' and ctx0 == 'OUTDENT')

	def queueScope val
		@scopes[@indents:length] = val
		self

	def popScope val
		@scopes.pop
		self

	def getScope
		@scopes[@indents:length - 1]

	def scope sym, opts
		var len = @ends.push(@end = sym)
		@contexts.push(opts or null)
		return sym

	def closeSelector
		if @end == '%'
			token('SELECTOR_END','%',0)
			pair('%')

	def openDef
		pushEnd('DEF')

	def closeDef
		if context == 'DEF'
			var prev = last(@tokens)

			if tT(prev) == 'TERMINATOR'
				let n = @tokens.pop
				token('DEF_BODY', 'DEF_BODY',0)
				@tokens.push(n)
			else
				token('DEF_BODY', 'DEF_BODY',0)

			pair('DEF')
		return

	def tagContextToken
		let chr = @chunk[0]
		let chr2 = @chunk[1]

		# let m = TAG_PART.exec(@chunk)
		let m = /^([A-Za-z\_\-\$\%\#][\w\-\$]*(\:[A-Za-z\_\-\$]+)*)/.exec(@chunk) # (\:[A-Za-z\_][\w\-]*)

		if m # and false
			let tok = m[1]
			let typ = 'TAG_LITERAL'
			let len = m[0]:length

			if tok == 'self' and @lastVal == '<'
				typ = 'SELF'

			if chr == '$' and (@lastTyp == 'TAG_TYPE' or @lastTyp == 'TAG_START')
				typ = 'TAG_REF'

			if chr == '%'
				typ = 'CSS_MIXIN'

			if chr == '#'
				typ = 'TAG_SYMBOL_ID'
				if tok:length == 1
					return 0

			token(typ,tok,len)
			return len

		if chr == '/' and chr2 == '>'
			token("TAG_END",'/>',2)
			pair 'TAG_END'
			return 2

		if chr == '%' or chr == ':' or chr == '.' or chr == '@'
			token("T{chr}",chr,1)

			if chr == '.' and (!chr2 or TSC_CARET_BEFORE[chr2]) and @platform == 'tsc'
				token('TAG_LITERAL','$CARET$',0,1)
			return 1

		elif chr == ' ' or chr == '\n' or chr == '\t'
			# add whitespace inside tag
			let m = /^[\n\s\t]+/.exec(@chunk)
			token 'TAG_WS', m[0], m[0]:length
			return m[0]:length

		elif chr == '=' and @chunk[1] != '>'
			token '=','=',1
			pushEnd('TAG_ATTR',id: 'VALUE', pop: /^([\s\n\>]|\/\>)/)
			return 1
		return 0

	def tagDefContextToken
		# console.log "tagContextToken"
		if var match = TAG_TYPE.exec(@chunk)
			token 'TAG_TYPE', match[0], match[0]:length
			return match[0]:length

		if var match = TAG_ID.exec(@chunk)
			var input = match[0]
			token 'TAG_ID', input, input:length
			return input:length

		if @chunk[0] == '\n'
			pair('TAG')

		return 0

	def findTypeAnnotation str,autoend = no
		var stack = []
		var i = 0
		var replaces = []
		var ending = /[\=\n\ \t\.\,\:\+]/
		# could it not happen here?
		while i < str:length
			var chr = str.charAt(i)
			let end = stack[0]
			let instr = end == '"' or end == "'"

			if chr and chr == end
				stack.shift()
				if autoend and stack:length == 0
					break i++

			elif !end and (chr == ')' or chr == ']' or chr == '}' or chr == '>')
				break

			elif chr == '('
				stack.unshift(')')
			elif chr == '['
				stack.unshift(']')
			elif chr == '{'
				stack.unshift('}')
			elif chr == '<'
				stack.unshift('>')
			elif chr == '"'
				stack.unshift('"')
			elif chr == "'"
				stack.unshift("'")
			elif !end and ending.test(chr)
				break
			i++

		return null if i == 0
		return str.slice(0,i)

	def findBalancedSelector str
		var stack = []
		var i = 0
		var replaces = []
		# could it not happen here?
		while i < (str:length - 1)
			var letter = str.charAt(i)

			let end = stack[0]
			let instr = end == '"' or end == "'"

			if letter and letter == end
				stack.shift

			elif !instr and (letter == ')' or letter == ']' or letter == '}')
				console.log 'out of balance!!'
				break

			elif letter == '/'
				replaces.unshift([i,1,':'])

			elif letter == '(' and !instr
				stack.unshift(')')
			# elif letter == '{' and !instr
			# 	stack.unshift('}')
			elif letter == '[' and !instr
				stack.unshift(']')
			elif letter == '"'
				stack.unshift('"')
			elif letter == "'"
				stack.unshift("'")

			if !end and (letter == '=' or letter == '\n' or letter == '{')
				break

			if !end and letter == ' '
				let after = str.slice(i + 1)
				if STYLE_PROPERTY.exec(after)
					break

				if INLINE_COMMENT.exec(after)
					break

			# console.log 'try',letter,i,end,str.substr(i,5)
			i++

		return null if i == 0

		let sel = str.slice(0,i)
		if replaces:length
			sel = sel.split('')
			for repl in replaces
				sel.splice(*repl)
			sel = sel.join('')

		return sel

	def lexStyleRule offset = 0, force = no
		# when we meet = enter into style context?
		let chunk = offset ? @chunk.slice(offset) : @chunk
		let sel = findBalancedSelector(chunk)

		if sel or force
			let len = sel ? sel:length : 0
			token('CSS_SEL',sel or '',len,offset)
			let seltoken = @last
			let next = chunk[len]

			if next == '='
				len++

			# if @context
			#	@context:lexer = null

			@indents.push 1
			@outdebt = @indebt = 0
			token('INDENT',"1",0,1)
			pushEnd('OUTDENT',lexer: self:lexStyleBody,opener: seltoken, style: yes)
			@indent++
			return len

		return 0

	def lexStyleBody
		return 0 if @end == '%'
		# return 0
		let chr = @chunk[0]
		var m

		let styleprop = STYLE_PROPERTY.exec(@chunk)
		let ltyp = @lastTyp

		if !styleprop and @chunk.match(/^([\^\%\*\w\&\$\>\/\.\[\@\!]|\#[\w\-]|\:\:)/) and (ltyp == 'TERMINATOR' or ltyp == 'INDENT')
			let sel = findBalancedSelector(@chunk)
			return lexStyleRule(0) if sel

		if styleprop
			# what is the last one?
			token('CSSPROP', styleprop[0], styleprop[0]:length)
			return styleprop[0]:length

		if chr[0] == '#' and m = STYLE_HEX.exec(@chunk)
			let next = @chunk[m[0]:length]
			let typ = next == '(' ? 'COLORMIX' : 'COLOR'
			token typ, m[0], m[0]:length
			return m[0]:length

		if chr == '/' and !@last:spaced
			# console.log '!!!'
			token '/', chr, 1
			return 1

		if m = STYLE_NUMERIC.exec(@chunk)
			let len = m[0]:length
			let typ = 'NUMBER'

			if m[2] == '%'
				typ = 'PERCENTAGE'
				# if @chunk[len]
			elif m[2]
				typ = 'DIMENSION'

			if @lastTyp == 'COMPARE' and !@last:spaced
				yes
			token typ, m[0], len
			return len

		elif m = STYLE_URL.exec(@chunk)
			# console.log 'matching style url',m
			let len = m[0]:length
			token 'CSSURL', m[0], len
			return m[0]:length

		elif m = STYLE_IDENTIFIER.exec(@chunk)
			let id = 'CSSIDENTIFIER'
			let val = m[0]
			let len = val:length
			if m[0].match(/^\-\-/)
				id = 'CSSVAR'
			elif @last and !@last:spaced and (ltyp == '}' or ltyp == ')')
				id = 'CSSUNIT'

			if @chunk[len] == '('
				id = 'CSSFUNCTION'

			token id, val, len

			return len

		elif @last and !@last:spaced and (ltyp == '}' or ltyp == ')') and chr == '%'
			token 'CSSUNIT', chr, 1
			return 1

		return 0

	def importsToken
		if var match = IMPORTS.exec(@chunk)
			token('IMPORTS',match[1],match[1]:length,7)
			return match[0]:length
		return 0

	def tagToken
		return 0 unless var match = TAG.exec(@chunk)
		var [input, type, identifier] = match


		if type == '<'
			if TYPE_GENERICS_AFTER.test(prevChars(1) or '')
				return 0
			# if @last and !@last:spaced and @lastTyp != 'TERMINATOR' and @lastTyp != 'INDENT'
			#	return 0

			token('TAG_START', '<',1)
			pushEnd(INVERSES['TAG_START'])

			if match = TAG_TYPE.exec(@chunk.substr(1,40))
				let next = @chunk[match[0]:length + 1]

				if match[0] != 'self' and (next != '{' and next != '-')
					token('TAG_TYPE',match[0],match[0]:length,1)
					return input:length + match[0]:length
			elif @chunk[1] == '>'
				token('TAG_TYPE','fragment',0,0)

			if identifier
				if identifier.substr(0,1) == '{'
					return type:length
				else
					token('TAG_NAME', input.substr(1),0)

		return input:length

	def selectorToken
		var match

		# special handling if we are in this context
		if @end == '%'
			var chr = @chunk[0]
			var ctx = @context

			var i = 0
			var part = ''
			var ending = no

			while chr = @chunk[i++]
				if chr == ')' and ctx:parens == 0
					ending = yes
					break

				elif chr == '('
					ctx:parens++
					part += '('
				elif chr == ')'
					ctx:parens--
					part += ')'
				elif chr == '{'
					break
				else
					part += chr

			if part
				token('SELECTOR_PART',part,i - 1)
			if ending
				token('SELECTOR_END',')',1,i - 1)
				pair '%'
				return i
			return i - 1

		return 0 unless match = SELECTOR.exec(@chunk)

		var [input, id, kind] = match

		# this is a closed selector
		if kind == '('
			# token '(','('
			token 'SELECTOR_START', id, id:length + 1
			pushEnd('%', parens: 0)
			return id:length + 1

		elif id == '%'
			# we are already scoped in on a selector
			return 1 if context == '%'
			token 'SELECTOR_START', id, id:length
			# this is a separate - scope. Full selector should rather be $, and keep the single selector as %

			pushEnd('%', open: yes)
			# @ends.push '%'
			# make sure a terminator breaks out
			return id:length
		else
			return 0

	def inTag
		var len = @ends:length
		if len > 0
			var ctx0 = @ends[len - 1]
			var ctx1 = len > 1 ? @ends[len - 2] : ctx0
			return ctx0 == 'TAG_END' or (ctx1 == 'TAG_END' and ctx0 == 'OUTDENT')
		return false

	def isKeyword id, next = ''
		if id == 'mixin' and (!next or next == ' ')
			return no if MEMBER_KEYWORDS[@lastTyp]
			return yes

		if @lastTyp == 'ATTR' or @lastTyp == 'PROP' or @lastTyp == 'DEF'
			return false

		# hack to allow imba.when to be exported with tree-shaking
		if id == 'when' and @lastTyp == 'CONST'
			return false

		if id == 'get' or id == 'set'
			if let m = @chunk.match(/^[gs]et ([\$\w\-]+|\[)/) # ( (do)|\n(\t+))
				let ctx = @contexts[@contexts:length - 1] or {}
				let before = ctx:opener and @tokens[@tokens.indexOf(ctx:opener) - 1]

				if @lastTyp in ['TERMINATOR','INDENT']
					if before and (before.@type == '=' or before.@type == '{')
						return true

		if (id == 'guard' or id == 'alter' or id == 'watch') and (getScope == 'PROP')
			# TODO Remove
			return true

		if id == 'css'
			# experimental css inside tag trees - making css keyword everywhere
			return true

			if (@lastTyp in ['TERMINATOR'] or !@lastTyp)
				return true

			if (@lastVal in ['global','local','export','default'])
				return true

			if (@lastTyp in ['='])
				return true
			
		if id == 'interface'
			if (@lastVal in ['global','export','default','declare'])
				return true

		if (id == 'attr' or id == 'prop' or id == 'get' or id == 'set' or id == 'css' or id == 'constructor' or id == 'declare')

			var scop = getScope
			var incls = scop == 'CLASS' or scop == 'TAG' or scop == 'EXTEND'

			# if id == 'css' and !@context and (@lastTyp in ['TERMINATOR'] or !@lastTyp)
			# 	return true

			# if id == 'css' and (@lastVal in ['global','local','export'])
			# 	return true

			if id == 'declare'
				return incls and @lastTyp in ['INDENT','TERMINATOR','DECORATOR']

			if id == 'constructor'
				return incls and @lastTyp in ['INDENT','TERMINATOR','DECORATOR']

			return true if incls

		ALL_KEYWORDS.indexOf(id) >= 0

	# Matches identifying literals: variables, keywords, method names, etc.
	# Check to ensure that JavaScript reserved words aren't being used as
	# identifiers. Because Imba reserves a handful of keywords that are
	# allowed in JavaScript, we're careful not to tokid them as keywords when
	# referenced as property names here, so you can still do `jQuery.is()` even
	# though `is` means `===` otherwise.
	def identifierToken
		var match

		var ctx0 = @ends:length > 0 ? @ends[@ends:length - 1] : null
		var ctx1 = @ends:length > 1 ? @ends[@ends:length - 2] : null
		var innerctx = ctx0
		var typ
		var reserved = no

		var addLoc = false
		var inTag = ctx0 == 'TAG_END' or (ctx1 == 'TAG_END' and ctx0 == 'OUTDENT')

		unless match = IDENTIFIER.exec(@chunk)
			return 0

		var [input, id, typ, m3, m4, colon] = match
		var idlen = id:length

		# What is the logic here?
		if id is 'own' and lastTokenType == 'FOR'
			token 'OWN', id, id:length
			return id:length

		var prev = last(@tokens)
		var lastTyp = @lastTyp

		if lastTyp == '#'
			token('IDENTIFIER', id, idlen)
			return idlen

		var forcedIdentifier = colon || lastTyp == '.' || lastTyp == '?.'

		forcedIdentifier = no if colon and lastTyp == '?' # for ternary

		# if we are not at the top level? -- hacky
		if id == 'tag' and @chunk.indexOf("tag(") == 0 # @chunk.match(/^tokid\(/)
			forcedIdentifier = yes

		if id == 'css' and (/css\s\:\:/).exec(@chunk)
			input = id + ' '
			colon = null
			forcedIdentifier = no

		var isKeyword = no

		# little reason to check for this right here? but I guess it is only a simple check
		if typ == '$' and ARGVAR.test(id)
			# console.log "TYP $"
			# if id == '$0'
			# 	typ = 'ARGUMENTS'
			# else
			typ = 'ARGVAR'
			id = id.substr(1)

		elif typ == '$' and ENV_FLAG.test(id)
			typ = 'ENV_FLAG'
			id = id.toUpperCase # .slice(1, -1)

		# elif typ == '#'
		#	typ = 'IVAR'

		elif typ == '@'
			if lastTyp == '.'
				typ = 'IDENTIFIER'
			else
				typ = 'DECORATOR'
		elif typ == '#'
			typ = 'SYMBOLID'
		elif typ == '##'
			typ = 'SYMBOLID'

		elif typ == '%'
			# use for something else
			let ltyp = @lastTyp
			if ltyp == 'TERMINATOR' or ltyp == 'INDENT' or ltyp == 'EXPORT'
				token('CSS', id,0)
				queueScope('CSS')
				return lexStyleRule(0,true)
			typ = 'CSS_MIXIN'

		elif typ == '$' and !colon
			typ = 'IDENTIFIER'

		elif id == 'elif' and !forcedIdentifier
			token 'ELSE', 'elif', id:length
			token 'IF', 'if'
			return id:length
		else
			typ = 'IDENTIFIER'

		# this catches all
		if !forcedIdentifier and isKeyword = self.isKeyword(id,@chunk[id:length])
			# (id in JS_KEYWORDS or id in IMBA_KEYWORDS)

			if typeof isKeyword == 'string'
				typ = isKeyword
			else
				typ = id.toUpperCase

			addLoc = true

			if typ == 'MODULE'
				if !(/^module [a-zA-Z]/).test(@chunk) or ctx0 == 'TAG_ATTR'
					typ = 'IDENTIFIER'

			# clumsy - but testing performance
			if typ == 'YES'
				typ = 'TRUE'
			elif typ == 'NO'
				typ = 'FALSE'
			elif typ == 'NIL'
				typ = 'NULL'
			elif (typ == 'MIXIN' or typ == 'INTERFACE')
				typ = 'CLASS'

			elif typ == 'VAR' or typ == 'CONST' or typ == 'LET'
				let ltyp = @lastTyp

				# extremely flakey - comma separated declaration blocks are inherently
				# ambiguous in the current syntax
				# if ltyp != 'TERMINATOR' and ltyp != 'INDENT' and ltyp != 'EXPORT' and ltyp
				#	typ = "INLINE_{typ}"

				# if @lastVal == 'export'
				#	tTs(prev,'EXPORT_VAR')

			# skipping
			elif typ == 'IF' or typ == 'ELSE' or typ == 'TRUE' or typ == 'FALSE' or typ == 'NULL'
				true

			elif typ == 'TAG'
				self.pushEnd('TAG')
			# FIXME @ends is not used the way it is supposed to..
			# what we want is a context-stack
			elif (typ == 'DEF' or typ == 'GET' or typ == 'SET')
				typ = 'DEF'
				openDef

			elif (typ == 'CONSTRUCTOR')
				token('DEF','',0)
				typ = 'IDENTIFIER'
				openDef

			elif typ == 'DO'
				closeDef if context == 'DEF'

			elif typ is 'WHEN' and LINE_BREAK.indexOf(lastTokenType) >= 0
				typ = 'LEADING_WHEN'

			elif typ is 'FOR'
				@seenFor = yes

			elif typ is 'UNLESS'
				typ = 'IF' # WARN

			elif UNARY.indexOf(typ) >= 0
				typ = 'UNARY'

			elif RELATION.indexOf(typ) >= 0
				if typ != 'INSTANCEOF' and typ != 'ISA' and @seenFor
					typ = 'FOR' + typ # ?
					@seenFor = no
				else
					typ = 'RELATION'

					if prev.@type == 'UNARY'
						prev.@type = 'NOT'

		# do we really want to check this here
		if !forcedIdentifier
			# should already have dealt with this

			if @lastVal == 'export' and id == 'default'
				# console.log 'id is default!!!'
				tTs(prev,'EXPORT')
				typ = 'DEFAULT'

			# these really should not go here?!?
			switch id
				when '!','not'                            then typ = 'UNARY'
				when '==', '!=', '===', '!==','is','isnt' then typ = 'COMPARE'
				when '&&', '||','and','or','??'           then typ = 'LOGIC'
				when 'super','break', 'continue', 'debugger','arguments' then typ = id.toUpperCase
				# when 'true', 'false', 'null', 'undefined' then typ = 'BOOL'
				# really?

		# prev = last @tokens
		var len = input:length

		# if typ == 'CONST' or typ == 'LET'
		#	if @lastVal == 'global' or @lastVal == 'declare'
		#		tTs(prev,@lastVal.toUpperCase)


		# should be strict about the order, check this manually instead
		if typ == 'CLASS' or typ == 'DEF' or typ == 'TAG' or typ == "PROP" or typ == 'CSS'
			queueScope(typ)

		if isKeyword and CONTEXTUAL_KEYWORDS[typ]	
			var i = @tokens:length
			var alts = CONTEXTUAL_KEYWORDS[typ]

			while i
				var prev = @tokens[--i]
				var ctrl = "" + tV(prev)
				if alts[ctrl]
					tTs(prev,ctrl.toUpperCase)
				else
					break

		elif typ == 'IF'
			queueScope(typ)

		elif typ == 'EXTEND' and !@chunk.match(/^extend (class|tag|interface|mixin)(\s|\n|$)/)
			queueScope(typ)

		elif typ == 'IMPORT'
			# console.log 'import last type',lastTyp,@chunk[idlen]
			let next = @chunk[idlen]
			if lastTyp == 'AWAIT' or next == '(' or next == '.'
				typ = 'IDENTIFIER'
			else
				pushEnd('IMPORT')
				token(typ, id, idlen)
				# return importsToken or len
				return len

		elif id == 'type' and lastTyp == 'IMPORT'
			token('TYPEIMPORT', id, idlen)
			# return importsToken or len
			return len

		elif typ == 'EXPORT'
			pushEnd('EXPORT')
			token(typ, id, idlen)
			return len

		elif id == 'from' and ctx0 == 'IMPORT'
			typ = 'FROM'
			pair 'IMPORT'

		elif id == 'from' and ctx0 == 'EXPORT'
			typ = 'FROM'
			pair 'EXPORT'

		elif id == 'as' and (ctx0 == 'IMPORT' or @lastTyp == 'IDENTIFIER' or ctx0 == 'EXPORT')
			typ = 'AS'

		# if id == 'new'
		#	console.log 'new keyword', @chunk.slice(0,5),@chunk.match(/^new\s+[\w\$\(]/)

		if id == 'new' and (@lastTyp != '.' and @chunk.match(/^new\s+[\w\$\(\<\#]/)) and @lastTyp != 'DEF'
			typ = 'NEW'

		if typ == 'IDENTIFIER'
			# see if previous was catch -- belongs in rewriter?
			if lastTyp == 'CATCH'
				typ = 'CATCH_VAR'

			if @lastVal == 'protected' and lastTyp == 'IDENTIFIER'
				tTs(prev,'PROTECTED')

		if (lastTyp == 'NUMBER' or lastTyp == ')') and !prev:spaced and (typ == 'IDENTIFIER' or id == '%')
			typ = 'UNIT'

		if colon
			token(typ, id, idlen)
			var colonOffset = colon.indexOf(':')

			moveCaret(idlen + colonOffset)

			token(':', ':',1)
			moveCaret(-(idlen + colonOffset))
			
		else
			token(typ, id, idlen)

		if typ == 'CSS'
			return len + lexStyleRule(len,true)



		return len

	# Matches numbers, including decimals, hex, and exponential notation.
	# Be careful not to interfere with ranges-in-progress.
	def numberToken
		var match, number, lexedLength

		return 0 unless match = NUMBER.exec(@chunk)

		number = match[0]
		lexedLength = number:length

		if var binaryLiteral = /0b([01_]+)/.exec(number)

			number = "" + parseInt(binaryLiteral[1].replace(/_/g,''), 2)

		var prev = last(@tokens)

		if match[0][0] == '.' && prev && !prev:spaced && ['IDENTIFIER',')','}',']','NUMBER'].indexOf(tT(prev)) >= 0
			# console.log "got here"
			token ".","."
			number = number.substr(1)

		token('NUMBER',number,lexedLength)
		return lexedLength

	def symbolToken
		var match, symbol, prev
		return 0 unless match = SYMBOL.exec(@chunk)
		symbol = match[0]
		prev = last(@tokens)

		if !prev or prev:spaced or @prevVal in ['(','[','=']
			let sym = helpers.dashToCamelCase(symbol.slice(1))
			token 'STRING', '"' + sym + '"', match[0]:length
			return match[0]:length
		return 0

	def escapeStr str, heredoc, q
		str = str.replace MULTILINER, (heredoc ? '\\n' : '')
		if q
			var r = RegExp("\\\\[{q}]","g")
			str = str.replace(r,q)
			str = str.replace RegExp("{q}","g"), '\\$&'
		return str

		# str = str.replace(MULTILINER, '\\n')
		# str = str.replace(/\t/g, '\\t')
	# Matches strings, including multi-line strings. Ensures that quotation marks
	# are balanced within the string's contents, and within nested interpolations.
	def stringToken
		var match, string

		switch @chunk.charAt(0)
			when "'"
				return 0 unless match = SIMPLESTR.exec(@chunk)
				string = match[0]
				token 'STRING', escapeStr(string), string:length
				# token 'STRING', (string = match[0]).replace(MULTILINER, '\\\n'), string:length

			when '"'
				return 0 unless string = balancedString(@chunk, '"')
				# what about tripe quoted strings?

				if string.indexOf('{') >= 0
					var len = string:length
					# if this has no interpolation?
					# we are now messing with locations - beware
					token 'STRING_START', string.charAt(0), 1
					interpolateString(string.slice 1, -1)
					token 'STRING_END', string.charAt(len - 1), 1, string:length - 1
				else
					var len = string:length
					# string = string.replace(MULTILINER, '\\\n')
					token 'STRING', escapeStr(string), len
			when '`'
				return 0 unless string = balancedString(@chunk, '`')

				if string.indexOf('{') >= 0
					var len = string:length
					# if this has no interpolation?
					# we are now messing with locations - beware
					token('STRING_START', string.charAt(0), 1)
					interpolateString(string.slice(1, -1),heredoc: true)
					token 'STRING_END', string.charAt(len - 1), 1, string:length - 1
				else
					var len = string:length
					# string = string.replace(MULTILINER, '\\\n')
					token 'STRING', escapeStr(string,true), len
			else
				return 0

		moveHead(string)
		return string:length

	# Matches heredocs, adjusting indentation to the correct level, as heredocs
	# preserve whitespace, but ignore indentation to the left.
	def heredocToken
		var match, heredoc, quote, doc

		return 0 unless match = HEREDOC.exec(@chunk)

		heredoc = match[0]
		quote = heredoc.charAt 0
		var opts = {quote: quote, indent: null, offset: 0}
		doc = sanitizeHeredoc(match[2], opts)
		# doc = match[2]
		# console.log "found heredoc {match[0]:length} {doc:length}"

		if quote == '"' && doc.indexOf('{') >= 0
			var open = match[1]
			# console.log doc.substr(0,3),match[1]
			# console.log 'heredoc here',open:length,open

			token 'STRING_START', open, open:length
			interpolateString(doc, heredoc: yes, offset: (open:length + opts:offset), quote: quote, indent: opts:realIndent)
			token 'STRING_END', open, open:length, heredoc:length - open:length
		else
			token('STRING', makeString(doc, quote, yes), 0)

		moveHead(heredoc)
		return heredoc:length

	def parseMagicalOptions str
		if str.indexOf('imba$') >= 0
			str.replace(/imba\$(\w+)\=(\S*)\b/g) do |m,name,val|
				if (/^\d+$/).test(val)
					val = parseInt(val)
				@opts[name] = val
		self

	# Matches and consumes comments.
	def commentToken
		var match, length, comment, indent, prev

		var typ = 'HERECOMMENT'

		if match = JS_COMMENT.exec(@chunk)
			token 'HERECOMMENT', match[1], match[1]:length
			token 'TERMINATOR', '\n'
			return match[0]:length

		if match = INLINE_COMMENT.exec(@chunk) # .match(INLINE_COMMENT)
			# console.log "match inline comment"
			length = match[0]:length
			indent = match[1]
			comment = match[2]
			let commentBody = (match[4] or '')
			if comment[0] == '#'
				commentBody = ' ' + commentBody

			prev = last(@tokens)
			var pt = prev and tT(prev)
			var note = '//' + commentBody # comment.substr(1)

			parseMagicalOptions(note)

			if @last and @last:spaced
				note = ' ' + note
				# console.log "the previous node was SPACED"
			# console.log "comment {note} - indent({indent}) - {length} {comment:length}"
			if note.match(/^\/\/ \@(type|param)/)
				note = '/**' + commentBody + '*/'
			elif note.match(/^\/\/ \<(reference)/)
				note = '///' + commentBody

			if (pt and pt != 'INDENT' and pt != 'TERMINATOR') or !pt
				# console.log "skip comment"
				# token 'INLINECOMMENT', comment.substr(2)
				# console.log "adding as terminator"
				token('TERMINATOR', note, length) # + '\n'
			else
				if pt == 'TERMINATOR'
					tVs(prev,tV(prev) + note)
					# prev[1] += note
				elif pt == 'INDENT'
					addLinebreaks(1,note)
				else
					# console.log "comment here"
					# should we ever get here?
					token(typ, comment.substr(2), length) # are we sure?

			return length # disable now while compiling

		# should use exec?
		return 0 unless match = COMMENT.exec(@chunk)

		var comment = match[0]
		var here = match[1]

		if here
			token 'HERECOMMENT', sanitizeHeredoc(here, herecomment: true, indent: Array(@indent + 1).join(' ')), comment:length
			token 'TERMINATOR', '\n'
		else
			token 'HERECOMMENT', comment, comment:length
			token 'TERMINATOR', '\n' # auto? really?

		moveHead(comment)
		return comment:length

	# Matches regular expression literals. Lexing regular expressions is difficult
	# to distinguish from division, so we borrow some basic heuristics from
	# JavaScript and Ruby.
	def regexToken
		var match, length, prev

		return 0 if @chunk.charAt(0) != '/'

		if match = HEREGEX.exec(@chunk)
			length = heregexToken(match)
			moveHead(match[0])
			return length

		prev = last @tokens
		# FIX
		return 0 if prev and (tT(prev) in (if prev:spaced then NOT_REGEX else NOT_SPACED_REGEX))
		return 0 unless match = REGEX.exec(@chunk)
		var [m, regex, flags] = match

		token 'REGEX', "{regex}{flags}", m:length
		m:length

	# Matches multiline extended regular expressions.
	# The escaping should rather happen in AST - possibly as an additional flag?
	def heregexToken match
		var [heregex, body, flags] = match
		token 'REGEX', heregex, heregex:length
		return heregex:length

	# Matches newlines, indents, and outdents, and determines which is which.
	# If we can detect that the current line is continued onto the the next line,
	# then the newline is suppressed:
	#
	#     elements
	#       .each( ... )
	#       .map( ... )
	#
	# Keeps track of the level of indentation, because a single outdent token
	# can close multiple indents, so we need to know how far in we happen to be.
	def lineToken
		var match

		return 0 unless match = MULTI_DENT.exec(@chunk)

		var indent = match[0]
		var brCount = moveHead(indent)

		@seenFor = no
		# reset column as well?
		var prev = last @tokens, 1
		let whitespace = indent.substr(indent.lastIndexOf('\n') + 1)
		var noNewlines = self.unfinished

		if (/^\n#\s/).test(@chunk)
			addLinebreaks(1)
			return 0

		# decide the general line-prefix by the very first line with characters

		# if gutter is undefined - we create it on the very first chance we have
		if @state:gutter == undefined
			@state:gutter = whitespace

		# if we have a gutter -- remove it
		if var gutter = @state:gutter or @opts:gutter
			if whitespace.indexOf(gutter) == 0
				whitespace = whitespace.slice(gutter:length)

			elif @chunk[indent:length] === undefined
				yes
			else
				error('incorrect indentation')

			# should throw error otherwise?

		var size = whitespace:length

		if @opts:dropIndentation
			return size

		if size > 0
			# seen indent?

			unless @indentStyle
				@opts:indent = @indentStyle = whitespace
				@indentRegex = RegExp.new(whitespace, 'g')

			let indentSize = 0
			let offset = 0
			let offsetLoc = @loc

			while true
				let idx = whitespace.indexOf(@indentStyle,offset)
				if idx == offset
					indentSize++
					offset += @indentStyle['length']
				elif offset == whitespace:length
					break
				else
					# workaround to report correct location
					@loc += indent:length - whitespace:length
					let start = @loc
					token('INDENT', whitespace,whitespace:length)

					error("Use tabs for indentation",{
						offset: start + offset
						length: whitespace:length - offset
					})

			size = indentSize

		if (size - @indebt) == @indent
			@scopes:length = @indents:length

			if noNewlines
				suppressNewlines()
			else
				newlineToken(brCount,indent)
			return indent:length

		if size > @indent
			if noNewlines
				@indebt = size - @indent
				suppressNewlines
				return indent:length

			if inTag
				return indent:length

			var diff = size - @indent + @outdebt
			closeDef()

			var expectScope = @scopes[@indents:length]
			var immediate = last(@tokens)

			if immediate and tT(immediate) == 'TERMINATOR'
				tTs(immediate,'INDENT')
				# should add terminator inside indent?
				immediate.@meta ||= {pre: tV(immediate), post: ''}
				immediate:scope = expectScope
			else
				# console.log "set indent {expectScope}"
				token('INDENT', "" + diff,0)
				@last:scope = expectScope

			@indents.push diff
			pushEnd('OUTDENT',opener: @last)
			@outdebt = @indebt = 0
			addLinebreaks(brCount)
		elif true
			@indebt = 0

			let moveOut = @indent - size
			let currIndent = @indent
			let useTabs = @indentStyle == '\t'
			let lines = indent.replace().split('\n')

			let levels = []
			let k = lines:length
			let lvl = 0
			while k > 0
				let ln = lines[--k]
				let lnlvl = useTabs ? ln:length : ln.replace(@indentRegex,'\t'):length
				if lnlvl > lvl
					lvl = lnlvl

				levels[k] = lvl

			levels[0] = currIndent

			# TODO track position as well

			let i = 0
			let toks = []
			let pre = ""
			for ln,idx in lines
				let lvl = levels[idx]

				while currIndent > lvl
					if pre
						terminatorToken(pre)
						pre = ""
					else
						terminatorToken('')

					moveOut--
					outdentToken(1,yes)
					currIndent--

				pre += '\n' + ln

			if pre
				terminatorToken(pre)

			while moveOut > 0
				outdentToken(1,yes)
				moveOut--
		else
			@indebt = 0
			outdentToken(@indent - size, noNewlines, brCount)
			addLinebreaks(brCount - 1)

		@indent = size
		return indent:length

	# Record an outdent token or multiple tokens, if we happen to be moving back
	# inwards past several recorded indents.
	def outdentToken moveOut, noNewlines, newlineCount
		# here we should also take care to pop / reset the scope-body
		# or context-type for indentation
		# console.log 'outdent!',@chunk,@loc,context == 'DEF'
		if context == 'DEF'
			closeDef

		var dent = 0
		while moveOut > 0
			var len = @indents:length - 1
			if @indents[len] == undefined
				moveOut = 0
			elif @indents[len] == @outdebt
				moveOut -= @outdebt
				@outdebt = 0
			elif @indents[len] < @outdebt
				@outdebt -= @indents[len]
				moveOut  -= @indents[len]
			else
				dent = @indents.pop - @outdebt
				moveOut -= dent
				@outdebt = 0

				addLinebreaks(1) unless noNewlines

				let paired = pair('OUTDENT')
				token('OUTDENT', "" + dent, 0)
				if paired[1] and paired[1]:opener
					let opener = paired[1]:opener

					@last.@opener = opener
					opener.@closer = @last
					if opener.@type == 'CSS_SEL'
						token('CSS_END', "", 0)
					# console.log 'paired now!!',opener,@tokens.indexOf(opener),@tokens:length - 1

		@outdebt -= moveOut if dent

		@tokens.pop while lastTokenValue == ';'

		token('TERMINATOR','\n',0) unless lastTokenType == 'TERMINATOR' or noNewlines
		# capping scopes so they dont hang around
		@scopes:length = @indents:length
		
		closeDef
		var ctx = context
		pair(ctx) if ctx == '%' or ctx == 'TAG' or ctx == 'IMPORT' or ctx == 'EXPORT' # or ctx == 'CSS' # really?
		return this

	# Matches and consumes non-meaningful whitespace. tokid the previous token
	# as being "spaced", because there are some cases where it makes a difference.
	def whitespaceToken type
		var match, nline, prev
		return 0 unless (match = WHITESPACE.exec(@chunk)) || (nline = @chunk.charAt(0) is '\n')
		prev = last @tokens

		# FIX - why oh why?
		if prev
			if match
				prev:spaced = yes
				# prev.@s = match[0]
				# console.log 'whitespace',JSON.stringify(match[0]),prev
				return match[0]:length
			else
				prev:newLine = yes
				return 0

	def moveHead str
		var br = count(str,'\n')
		return br

	def terminatorToken content, loc
		if @lastTyp == 'TERMINATOR'
			@last.@value += content
			# add location info as well?
		else
			token('TERMINATOR',content,loc)

	def addLinebreaks count, raw
		var br

		return this if !raw and count == 0 # no terminators?

		var prev = @last

		if !raw
			if count == 1
				br = '\n'
			elif count == 2
				br = '\n\n'
			elif count == 3
				br = '\n\n\n'
			else
				br = repeatString('\n',count)
		# FIX
		if prev
			var t = prev.@type # @lastTyp
			var v = tV(prev)

			# we really want to add this
			if t == 'INDENT'
				# TODO we want to add to the indent
				# console.log "add the comment to the indent -- pre? {raw} {br}"

				var meta = prev.@meta ||= {pre: '', post: ''}
				meta:post += (raw or br)
				# tVs(v + (raw or br))
				return this

			elif t == 'TERMINATOR'
				# console.log "already exists terminator {br} {raw}"
				tVs(prev,v + (raw or br))
				return this

		token('TERMINATOR', (raw or br), 0)
		return

	# Generate a newline token. Consecutive newlines get merged together.
	def newlineToken lines, raw

		# while lastTokenValue == ';'
		#	@tokens.pop

		addLinebreaks(lines,raw)
		# WARN now import cannot go over multiple lines
		closeDef()  # close def -- really?
		var ctx = context
		pair(ctx) if ctx == 'TAG' or ctx == 'IMPORT' or ctx == 'EXPORT'
		# closeDef()  # close def -- really?
		this

	# Use a `\` at a line-ending to suppress the newline.
	# The slash is removed here once its job is done.
	def suppressNewlines
		@tokens.pop if value() is '\\'
		this

	# We treat all other single characters as a token. E.g.: `( ) , . !`
	# Multi-character operators are also literal tokens, so that Jison can assign
	# the proper order of operations. There are some symbols that we tokid specially
	# here. `;` and newlines are both treated as a `TERMINATOR`, we distinguish
	# parentheses that indicate a method call from regular parentheses, and so on.
	def literalToken
		var match, value
		if match = OPERATOR.exec(@chunk)
			value = match[0]
			tagParameters if CODE.test(value)
		else
			value = @chunk.charAt(0)

		var end1 = @ends[@ends:length - 1]
		var end2 = @ends[@ends:length - 2]

		var inTag = end1 == 'TAG_END' or end1  == 'OUTDENT' and end2 == 'TAG_END'

		var tokid = value
		var prev  = last @tokens
		var pt = prev and tT(prev)
		var pv = prev and tV(prev)
		var length = value:length

		# is this needed?
		if value == '=' and prev

			if pv == '||' or pv == '&&' # in ['||', '&&']
				tTs(prev,'COMPOUND_ASSIGN')
				tVs(prev,pv + '=') # need to change the length as well
				prev.@len = @loc - prev.@loc + value:length
				return value:length

		if value == 'ƒ'
			tokid = 'DO'

		if value == '|'
			# hacky way to implement this
			# with new lexer we'll use { ... } instead, and assume object-context,
			# then go back and correct when we see the context is invalid
			if pv == '('
				token('DO', 'DO',0)
				self.pushEnd('|')
				token('BLOCK_PARAM_START', value,1)
				return length

			elif pt == 'DO'
				self.pushEnd('|')
				token('BLOCK_PARAM_START', value,1)
				return length

			elif end1 == '|'
				token('BLOCK_PARAM_END', value,1)
				pair '|'
				return length

		if value is ';'
			@seenFor = no
			tokid = 'TERMINATOR'

		if value == '(' and pt == 'T.'
			tokid = 'STYLE_START'

		elif value == '[' and inTag
			tokid = 'STYLE_START'

		elif value is '(' and inTag and pt != '=' and prev:spaced # FIXed
			# console.log 'spaced before ( in tokid'
			# FIXME - should rather add a special token like TAG_PARAMS_START
			token ',',','

		elif value is '->' and inTag
			tokid = 'TAG_END'
			pair 'TAG_END'

		elif value is '=>' and inTag
			tokid = 'TAG_END'
			pair 'TAG_END'

		elif value is '/>' and inTag
			tokid = 'TAG_END'
			pair 'TAG_END'

		elif value is '>' and inTag
			tokid = 'TAG_END'
			pair 'TAG_END'

		elif value is 'TERMINATOR' and end1 is 'DEF'
			closeDef()

		# TODO BLOCK PARAM BUG
		# really+
		elif value is '&' and context == 'DEF'
			# console.log("okay!")
			tokid = 'BLOCK_ARG'
			# change the next identifier instead?
		elif value == '---'
			tokid = 'SEPARATOR'
		elif value == '-' and pt == 'TERMINATOR' and @chunk.match(/^\-\s*\n/)
			tokid = 'SEPARATOR'

		# elif value.match()
		elif value == '*' and @chunk.charAt(1).match(/[A-Za-z\_\@\[]/) and (prev:spaced or [',','(','[','{','|','\n','\t'].indexOf(pv) >= 0)
			tokid = "SPLAT"

		elif value == '*' and (context == 'IMPORT' or context == 'EXPORT')
			tokid = "{context}_ALL"

		elif value == ',' and context == 'IMPORT'
			tokid = "IMPORT_COMMA"

		elif value == '!' and prev and !prev:spaced and ([']',')'].indexOf(pv) >= 0 or (pt == 'IDENTIFIER' or pt == 'SYMBOLID' or pt == 'SUPER'))
			tokid = 'BANG'

		elif value == '&' and @chunk.match(/^\&\s*[,\)\}\]]/)
			tokid = 'DO_PLACEHOLDER'
		elif value == '&' and @chunk.match(/^\&(\s*[\.\>\<\=\%\^]|\s+(is|isnt|in|not|isa)\b|[\[\(])/)
			tokid = 'AMPER_REF'
		elif value == '**'
			tokid = 'EXP'
		elif value == '%' and (pt == 'NUMBER' or pt == ')') and !prev:spaced
			tokid = 'UNIT'
		elif value in MATH
			tokid = 'MATH'
		elif value in COMPARE
			tokid = 'COMPARE'
		elif value in COMPOUND_ASSIGN
			tokid = 'COMPOUND_ASSIGN'
		elif value in UNARY
			tokid = 'UNARY'
		elif value in SHIFT
			tokid = 'SHIFT'
		elif value in LOGIC
			tokid = 'LOGIC' # or value is '?' and prev?:spaced

		elif prev and !prev:spaced
			if value == '{' and pt == 'IDENTIFIER'
				tokid = '{{'

			elif value == '{' and pt == '#'
				tokid = '{{'

			if value is '(' and pt in CALLABLE
				tokid = 'CALL_START'

			elif value is '(' and pt == 'DO'
				tokid = 'BLOCK_PARAM_START'

			elif value is '[' and pt in INDEXABLE
				tokid = 'INDEX_START'
				tTs(prev,'INDEX_SOAK') if pt == '?'

		if pv == '&' and pt != 'AMPER_REF'
			if !prev:spaced and tokid in ['COMPARE','.','(','[']
				tTs(prev,pt = 'AMPER_REF')
			elif prev:spaced and tokid in ['COMPARE']
				tTs(prev,pt = 'AMPER_REF')


		let opener = null

		switch value
			when '(', '{', '['
				pushEnd(INVERSES[value],closeType: INVERSES[tokid],i: @tokens:length)
			when ')', '}', ']'
				let paired = pair(value)
				if paired and paired[1]:closeType
					tokid = paired[1]:closeType
					let other = @tokens[paired[1]:i]
					opener = @tokens[paired[1]:i]

		if value == '\\'
			tokid = 'TYPE'
			let annotation = findTypeAnnotation(@chunk.slice(1))
			if annotation
				value = value + annotation

		# Add unspaced <> gemneric annotation?
		if value == '<' and TYPE_GENERICS_AFTER.test(prevChars(1))
			
			let typ = findTypeAnnotation(@chunk,yes)
			if typ
				tokid = 'GENERICS'
				value = typ

		if value == '..' and !prev:spaced
			tokid = '?.'
			value = '?.'

		if value == ':' and end1 == 'TAG_RULE'
			tokid = 'T:'

		if (tokid == '-' or tokid == '+')
			if /\w|\(|\$/.test(@chunk[1]) and (!prev or prev:spaced)
				tokid = tokid + tokid + tokid

		token(tokid, value, value:length)

		if opener
			opener.@closer = @last

		if @platform == 'tsc'
			let next = @chunk[1] or ''
			if value == '.' and (!next or TSC_CARET_BEFORE[next])
				token('IDENTIFIER','$CARET$',0,1)
			elif value == '@' and (!next or (/[^\$\@\-\.\w]/).test(next)) and false
				token('IDENTIFIER','$CARET$',0,1)

		return value:length

	# Token Manipulators
	# ------------------

	# Sanitize a heredoc or herecomment by
	# erasing all external indentation on the left-hand side.
	def sanitizeHeredoc doc, options
		var match
		var indent = options:indent
		var herecomment = options:herecomment

		if herecomment
			if HEREDOC_ILLEGAL.test(doc)
				error("block comment cannot contain '*/' starting")

			return doc if doc.indexOf('\n') <= 0
		else
			while match = HEREDOC_INDENT.exec(doc)
				var attempt = match[1]
				if indent is null or 0 < attempt:length < indent:length
					indent = attempt

		doc = doc.replace RegExp("\\n{indent}","g"), '\n' if indent
		unless herecomment
			if doc[0] == '\n'
				options:offset = indent:length + 1
			doc = doc.replace(/^\n/, '')
		options:realIndent = indent
		return doc

	# A source of ambiguity in our grammar used to be parameter lists in function
	# definitions versus argument lists in function calls. Walk backwards, tokidging
	# parameters specially in order to make things easier for the parser.
	def tagParameters
		return this if lastTokenType != ')'
		var stack = []
		var tokens = @tokens
		var i = tokens:length

		tTs(tokens[--i], 'PARAM_END')

		while var tok = tokens[--i]
			var t = tT(tok)
			switch t
				when ')'
					stack.push tok
				when '(', 'CALL_START'
					if stack:length
						stack.pop
					elif t is '('
						tTs(tok,'PARAM_START')
						return this
					else
						return this

		return this

	# Close up all remaining open blocks at the end of the file.
	def closeIndentation
		while true
			var ctx = context
			if ctx == 'TAG' or ctx == 'IMPORT' or ctx == 'EXPORT'
				pair(ctx)
			else
				break
		# pair(context) if context == 'IMPORT' or context == 'EXPORT'
		closeDef
		closeSelector
		outdentToken(@indent,no,0)

	# Matches a balanced group such as a single or double-quoted string. Pass in
	# a series of delimiters, all of which must be nested correctly within the
	# contents of the string. This method allows us to have strings within
	# interpolations within strings, ad infinitum.
	def balancedString str, end
		var match, letter, prev

		var stack = [end]
		var i = 0

		# could it not happen here?
		while i < (str:length - 1)
			i++
			var letter = str.charAt(i)
			switch letter
				when '\\'
					i++
					continue
				when end
					stack.pop
					unless stack:length
						var v = str.slice(0, i + 1)
						return v
					end = stack[stack:length - 1]
					continue

			if end is '}' and (letter == '"' or letter == "'" or letter == "`")
				stack.push(end = letter)

			elif end is '}' and letter is '/' and match = (HEREGEX.exec(str.slice i) or REGEX.exec(str.slice i))
				i += match[0]:length - 1

			elif end is '}' and letter is '{'
				stack.push end = '}'
			elif end is '"' and letter is '{'
				stack.push end = '}'
			elif end is '`' and letter is '{'
				stack.push end = '}'
			prev = letter

		error("missing { stack.pop }, starting")

	# Expand variables and expressions inside double-quoted strings using
	# braces for substitution of arbitrary expressions.
	#
	#     "Hello {name.capitalize}."
	#
	# If it encounters an interpolation, this method will recursively create a
	# new Lexer, tokenize the interpolated contents, and merge them into the
	# token stream.
	def interpolateString str, options = {}

		var heredoc = options:heredoc
		var quote = options:quote
		var regex = options:regex
		var prefix = options:prefix
		var indent = options:indent

		var startLoc = @loc
		var tokens = []
		var pi = 0
		var i  = -1
		var locOffset = options:offset or 1
		var strlen = str:length
		var letter
		var expr

		var isInterpolated = no
		# out of bounds

		while letter = str[i += 1]
			if letter is '\\'
				i += 1
				continue

			if letter is '\n' and indent
				locOffset += indent:length

			unless str[i] == '{' and (expr = balancedString(str.slice(i), '}'))
				continue

			isInterpolated = yes

			# these have no real sense of location or anything?
			if pi < i
				# this is the prefix-string - before any item
				var tok = Token.new('NEOSTRING', escapeStr(str.slice(pi, i),heredoc,quote),@loc + pi + locOffset,i - pi)
				# tok.@loc = @loc + pi
				# tok.@len = i - pi + 2
				tokens.push(tok)

			tokens.push Token.new('{{','{',@loc + i + locOffset,1)

			var inner = expr.slice(1, -1)

			# remove leading spaces
			# need to keep track of how much whitespace we dropped from the start
			inner = inner.replace(/^[^\n\S]+/,'')

			if inner:length
				# we need to remember the loc we start at
				# console.log('interpolate from loc',@loc,i)
				# really? why not just add to the stack??
				# what about the added
				# should share with the selector no?
				# console.log "tokenize inner parts of string",inner
				var spaces = 0
				var offset = @loc + i + (expr:length - inner:length) - 1
				# why create a whole new lexer? Should rather reuse one
				# much better to simply move into interpolation mode where
				# we continue parsing until we meet unpaired }
				var nested = Lexer.new.tokenize(inner, {inline: yes, rewrite: no, loc: offset + locOffset}, @script)
				# console.log nested.pop

				if nested[0] and tT(nested[0]) == 'TERMINATOR'
					nested.shift

				if nested:length
					tokens.push *nested # T.token('TOKENS',nested,0)

			# should rather add the amount by which our lexer has moved?
			i += expr:length - 1
			tokens.push Token.new('}}','}',@loc + i + locOffset,1)
			pi = i + 1

		# adding the last part of the string here
		if i >= pi and pi < str:length
			# set the length as well - or?
			# the string after?
			# console.log 'push neostring'
			tokens.push Token.new('NEOSTRING', escapeStr(str.slice(pi),heredoc,quote),@loc + pi + locOffset, str:length - pi)

		# console.log tokens:length
		return tokens if regex

		return token 'NEOSTRING', '""' unless tokens:length

		@tokens.push(tok) for tok in tokens

		return tokens

	# Matches a balanced group such as a single or double-quoted string. Pass in
	# a series of delimiters, all of which must be nested correctly within the
	# contents of the string. This method allows us to have strings within
	# interpolations within strings, ad infinitum.
	def balancedSelector str, end
		var prev
		var letter
		var stack = [end]
		# FIXME
		for i in [1...str:length]
			switch letter = str.charAt(i)
				when '\\'
					i++
					continue
				when end
					stack.pop
					unless stack:length
						return str.slice(0, i + 1)

					end = stack[stack:length - 1]
					continue
			if end is '}' and letter is ')'
				stack.push end = letter
			elif end is '}' and letter is '{'
				stack.push end = '}'
			elif end is ')' and letter is '{'
				stack.push end = '}'
			prev = letter # what, why?

		error("missing { stack.pop }, starting")

	# Pairs up a closing token, ensuring that all listed pairs of tokens are
	# correctly balanced throughout the course of the token stream.
	def pair tok
		var wanted = last(@ends)
		unless tok == wanted
			unless 'OUTDENT' is wanted
				error("unmatched {tok}", length: tok:length)
			var size = last(@indents)
			@indent -= size
			outdentToken(size, true, 0)
			return pair(tok)
		self.popEnd

	# Helpers
	# -------

	# Add a token to the results, taking note of the line number.
	def token id, value, len, offset
		@lastTyp = id
		@lastVal = value
		var tok = @last = Token.new(id, value, @loc + (offset or 0), len or 0)
		@tokens.push tok
		return

	def lastTokenType
		var token = @tokens[@tokens:length - 1]
		token ? tT(token) : 'NONE'

	def lastTokenValue
		var token = @tokens[@tokens:length - 1]
		token ? token.@value : ''

	# Peek at a tokid in the current token stream.
	def tokid index, val
		if var tok = last(@tokens, index)
			tTs(tok,val) if val
			return tT(tok)
		else null

	# Peek at a value in the current token stream.
	def value index, val
		if var tok = last(@tokens, index)
			tVs(tok,val) if val
			return tV(tok)
		else null

	# Are we in the midst of an unfinished expression?
	def unfinished
		return true if LINE_CONTINUER.test(@chunk) and (!@context or !@context:style)
		return (UNFINISHED.indexOf(@lastTyp) >= 0 and @platform != 'tsc')

	# Converts newlines for string literals.
	def escapeLines str, heredoc
		str.replace MULTILINER, (heredoc ? '\\n' : '')

	# Constructs a string token by escaping quotes and newlines.
	def makeString body, quote, heredoc
		return quote + quote unless body
		body = body.replace(/\\([\s\S])/g) do |match, contents|
			(contents == '\n' or contents == quote) ? contents : match
		# Does not work now
		body = body.replace RegExp("{quote}","g"), '\\$&'
		quote + escapeLines(body, heredoc) + quote

	# Throws a syntax error on the current `@line`.
	def error message, params = {} # len = 0
		let loc = params:offset or @loc
		let err = @script.addDiagnostic('error',{
			message: message
			source: params:source or 'imba-lexer'
			range: params:range or @script.rangeAt(loc,loc + (params:length or len))
		})
		throw err.toError