1 # $Id: scanner.rb,v 1.7 2011/11/30 19:26:34 machan Exp $
2
3 require 'irb/ruby-lex'
4 require 'irb/ruby-token'
5
6 require 'tmdoc/tmstd'
7 require 'tmdoc/constant'
8 require 'tmdoc/environment'
9
10
11 module TmDoc
12
13 module Reader::Ruby18
14
15 class Scanner
16 include RubyToken
17
18 attr_reader :lexer, :file_name
19
20
21 def initialize(input, env, file_name)
22 ASSERT.kind_of input, IO
23 ASSERT.kind_of env, ENV::Environment
24 ASSERT.kind_of file_name, String
25
26 @tokens = []
27 @readed = []
28 @unget_readed = []
29
30 @env = env
31
32 @lexer = RubyLex.new
33 @lexer.exception_on_syntax_error = false
34 @lexer.set_input input
35
36 @file_name = file_name
37 end
38
39
40 def lex_state
41 state = self.lexer.instance_eval { # tricky tech.
42 @lex_state
43 }
44
45 ASSERT.kind_of state, Symbol
46 end
47
48
49 def lex_state!(lex_state)
50 self.lexer.instance_eval { # tricky tech.
51 @lex_state = lex_state
52 }
53
54 nil
55 end
56
57
58 def get!
59 if @tokens.empty?
60 tk = @lexer.token
61 @readed.push @lexer.get_readed
62 else
63 @readed.push @unget_readed.shift
64 tk = @tokens.shift
65 end
66
67 case tk
68 when TkSYMBEG
69 tk1 = get!
70 tk =
71 case tk1
72 when TkId, TkOp, TkCONSTANT, TkIDENTIFIER, TkFID
73 TkSYMBOL.new(
74 tk1.seek, tk1.line_no, tk1.char_no,
75 ":" + tk1.name
76 )
77 when TkSTRING
78 TkSYMBOL.new(
79 tk1.seek, tk1.line_no, tk1.char_no,
80 ':"' + last_readed + '"'
81 )
82 else
83 if tk1.respond_to?(:name)
84 TkSYMBOL.new(
85 tk1.seek, tk1.line_no, tk1.char_no,
86 ":" + tk1.name
87 )
88 else
89 LOG::Notice.log(
90 "':' followed by unexpected token: <%s>\n" +
91 "\t-- #%d in '%s'",
92 tk1.class.to_s, tk1.line_no, @file_name
93 )
94 CMD.test_sensitive_level @env, LOG::Notice
95
96 tk1
97 end
98 end
99 else
100 ; # nop
101 end
102
103 if @env.debug_scanner?
104 LOG::Debug.log "[Debug.S] GOT --> %s\n", tk.inspect
105 end
106
107 ASSERT.opt_kind_of tk, RubyToken::Token
108 end
109
110
111 def peek
112 tk = get!
113 unget! tk
114
115 ASSERT.opt_kind_of tk, RubyToken::Token
116 end
117
118
119 def unget!(tk)
120 ASSERT.opt_kind_of tk, RubyToken::Token
121
122 @tokens.unshift tk
123 @unget_readed.unshift @readed.pop
124
125 nil
126 end
127
128
129 def skip_space!
130 tokens = []
131 while (tk = get!).kind_of?(TkSPACE)
132 tokens << tk
133 end
134 unget! tk
135
136 ASSERT.kind_of tokens, Array
137 end
138
139
140 def skip_space_or_nl!
141 tokens = []
142 while (tk = get!).kind_of?(TkSPACE) || tk.kind_of?(TkNL)
143 tokens << tk
144 end
145 unget! tk
146
147 ASSERT.kind_of tokens, Array
148 end
149
150
151 def get_readed!(&block)
152 if block
153 @readed = []
154 block.call
155 end
156
157 str = @readed.join ''
158 @readed = []
159
160 ASSERT.kind_of str, String
161 end
162
163 def last_readed
164 str = @readed[-1]
165
166 ASSERT.kind_of str, String
167 end
168 end
169
170 end # TmDoc::Reader::Ruby18
171
172 end # TmDoc