public interface ITokenCategorizer : ILanguageService, IRemotable {
	
	void Initialize (object state, Microsoft.Scripting.SourceUnitReader sourceReader, Microsoft.Scripting.SourceLocation initialLocation);
	TokenInfo ReadToken ();
	System.Collections.Generic.IEnumerable<TokenInfo> ReadTokens (int countOfChars);
	bool SkipToken ();
	bool SkipTokens (int countOfChars);
	
	Microsoft.Scripting.SourceLocation CurrentPosition {
		get;
	}
	object CurrentState {
		get;
	}
	ErrorSink ErrorSink {
		get;
		set;
	}
}

[Serializable]
public struct TokenInfo : IEquatable<TokenInfo> {
	
	public TokenInfo (Microsoft.Scripting.SourceSpan span, TokenCategory category, TokenTriggers trigger);
	
	public override bool Equals (object o);
	public bool Equals (TokenInfo other);
	protected virtual void Finalize ();
	public override int GetHashCode ();
	public Type GetType ();
	protected object MemberwiseClone ();
	public override string ToString ();
	
	public TokenCategory Category {
		get;
		set;
	}
	public Microsoft.Scripting.SourceSpan SourceSpan {
		get;
		set;
	}
	public TokenTriggers Trigger {
		get;
		set;
	}
}

[Serializable]
public enum TokenCategory {
	None,
	EndOfStream,
	WhiteSpace,
	Comment,
	LineComment,
	DocComment,
	NumericLiteral,
	CharacterLiteral,
	StringLiteral,
	RegularExpressionLiteral,
	Keyword,
	Directive,
	Operator,
	Delimiter,
	Identifier,
	Grouping,
	Error,
	LanguageDefined
}

[Serializable]
[Flags]
public enum TokenTriggers {
	None,
	MemberSelect,
	MatchBraces,
	ParameterStart,
	ParameterNext,
	ParameterEnd,
	Parameter,
	MethodTip
}

[Serializable]
public struct SourceLocation {
	
	public SourceLocation (int index, int line, int column);
	
	public static int Compare (SourceLocation left, SourceLocation right);
	public override bool Equals (object obj);
	protected virtual void Finalize ();
	public override int GetHashCode ();
	public Type GetType ();
	protected object MemberwiseClone ();
	public override string ToString ();
	
	public static bool operator == (SourceLocation left, SourceLocation right);
	public static bool operator != (SourceLocation left, SourceLocation right);
	public static bool operator < (SourceLocation left, SourceLocation right);
	public static bool operator > (SourceLocation left, SourceLocation right);
	public static bool operator <= (SourceLocation left, SourceLocation right);
	public static bool operator >= (SourceLocation left, SourceLocation right);
	
	public int Column {
		get;
	}
	public int Index {
		get;
	}
	public bool IsValid {
		get;
	}
	public int Line {
		get;
	}
	
	public static readonly SourceLocation None;
	public static readonly SourceLocation Invalid;
	public static readonly SourceLocation MinValue;
}

[Serializable]
public struct SourceSpan {
	
	public SourceSpan (SourceLocation start, SourceLocation end);
	
	public override bool Equals (object obj);
	protected virtual void Finalize ();
	public override int GetHashCode ();
	public Type GetType ();
	protected object MemberwiseClone ();
	public override string ToString ();
	
	public static bool operator == (SourceSpan left, SourceSpan right);
	public static bool operator != (SourceSpan left, SourceSpan right);
	
	public SourceLocation End {
		get;
	}
	public bool IsValid {
		get;
	}
	public int Length {
		get;
	}
	public SourceLocation Start {
		get;
	}
	
	public static readonly SourceSpan None;
	public static readonly SourceSpan Invalid;
}