128 lines
2.9 KiB
Smalltalk
128 lines
2.9 KiB
Smalltalk
|
"
|
||
|
|
I am capable of creating PPCTokenizingParserNode.
|
||
|
|
|
||
|
|
This is a node that
|
||
|
|
- has all all the tokens collected,
|
||
|
|
- identifies the whitespace
|
||
|
|
- replaces all the token reads with token consume
|
||
|
|
"
|
||
|
|
Class {
|
||
|
|
#name : 'PPCTokenizingVisitor',
|
||
|
|
#superclass : 'PPCRewritingVisitor',
|
||
|
|
#instVars : [
|
||
|
|
'tokens'
|
||
|
|
],
|
||
|
|
#category : 'PetitCompiler-Visitors'
|
||
|
|
}
|
||
|
|
|
||
|
|
{ #category : 'tokens' }
|
||
|
|
PPCTokenizingVisitor >> addToken: token [
|
||
|
|
(tokens contains: [:e | e = token] ) ifFalse: [
|
||
|
|
tokens addLast: token
|
||
|
|
]
|
||
|
|
]
|
||
|
|
|
||
|
|
{ #category : 'hooks' }
|
||
|
|
PPCTokenizingVisitor >> afterAccept: node retval: retval [
|
||
|
|
| newRetval |
|
||
|
|
self isRoot ifTrue: [
|
||
|
|
| tokensNode whitespaceNode |
|
||
|
|
|
||
|
|
tokens do: [ :token | token unmarkForInline ].
|
||
|
|
whitespaceNode := tokens detect: [ :e | e isTrimmingTokenNode ] ifNone: [ nil ].
|
||
|
|
whitespaceNode notNil ifTrue: [
|
||
|
|
whitespaceNode := whitespaceNode whitespace copy
|
||
|
|
unmarkForInline;
|
||
|
|
name: 'consumeWhitespace';
|
||
|
|
yourself.
|
||
|
|
"whitespaceNode := PPCTokenWhitespaceNode new
|
||
|
|
child: whitespaceNode;
|
||
|
|
yourself"
|
||
|
|
] ifFalse: [
|
||
|
|
whitespaceNode := PPCNilNode new
|
||
|
|
name: 'consumeWhitespace';
|
||
|
|
yourself
|
||
|
|
].
|
||
|
|
|
||
|
|
tokensNode := PPCListNode new
|
||
|
|
children: tokens asArray;
|
||
|
|
name: 'nextToken';
|
||
|
|
yourself.
|
||
|
|
|
||
|
|
|
||
|
|
newRetval := PPCTokenizingParserNode new
|
||
|
|
parser: retval;
|
||
|
|
whitespace: whitespaceNode;
|
||
|
|
tokens: tokensNode;
|
||
|
|
name: (retval name isNil ifTrue: [ #mainParser ] ifFalse: [ retval name ]);
|
||
|
|
yourself
|
||
|
|
] ifFalse: [
|
||
|
|
newRetval := retval
|
||
|
|
].
|
||
|
|
|
||
|
|
^ super afterAccept: node retval: newRetval
|
||
|
|
|
||
|
|
"Modified: / 12-05-2015 / 01:37:57 / Jan Vrany <jan.vrany@fit.cvut.cz>"
|
||
|
|
]
|
||
|
|
|
||
|
|
{ #category : 'hooks' }
|
||
|
|
PPCTokenizingVisitor >> eofToken [
|
||
|
|
| ws |
|
||
|
|
self error: 'deprecated?'.
|
||
|
|
ws := PPCStarNode new
|
||
|
|
child: (PPCMessagePredicateNode new
|
||
|
|
message: #isSeparator;
|
||
|
|
yourself);
|
||
|
|
yourself.
|
||
|
|
|
||
|
|
^ PPCTrimmingTokenNode new
|
||
|
|
child: PPCEndOfFileNode new;
|
||
|
|
whitespace: ws;
|
||
|
|
tokenClass: PPToken;
|
||
|
|
name: 'eof';
|
||
|
|
yourself.
|
||
|
|
]
|
||
|
|
|
||
|
|
{ #category : 'initialization' }
|
||
|
|
PPCTokenizingVisitor >> initialize [
|
||
|
|
super initialize.
|
||
|
|
tokens := OrderedCollection new.
|
||
|
|
]
|
||
|
|
|
||
|
|
{ #category : 'testing' }
|
||
|
|
PPCTokenizingVisitor >> isRoot [
|
||
|
|
^ openSet size = 1
|
||
|
|
]
|
||
|
|
|
||
|
|
{ #category : 'visiting' }
|
||
|
|
PPCTokenizingVisitor >> visitTokenConsumeNode: node [
|
||
|
|
"
|
||
|
|
Seems, it might happen, that if I create the consume node,
|
||
|
|
I will ge to it later. This would create a token consume node for the
|
||
|
|
child, thus having tokenConsumNode with tokenConsumNode as a child...
|
||
|
|
"
|
||
|
|
^ node
|
||
|
|
]
|
||
|
|
|
||
|
|
{ #category : 'visiting' }
|
||
|
|
PPCTokenizingVisitor >> visitTokenNode: node [
|
||
|
|
self addToken: node.
|
||
|
|
|
||
|
|
self assert: node acceptsEpsilon not description: 'Sorry, but the epsilon tokens are not allowed'.
|
||
|
|
|
||
|
|
^ PPCTokenConsumeNode new
|
||
|
|
child: node;
|
||
|
|
name: node name;
|
||
|
|
yourself.
|
||
|
|
]
|
||
|
|
|
||
|
|
{ #category : 'visiting' }
|
||
|
|
PPCTokenizingVisitor >> visitTrimmingTokenNode: node [
|
||
|
|
self addToken: node.
|
||
|
|
|
||
|
|
^ PPCTokenConsumeNode new
|
||
|
|
child: node;
|
||
|
|
name: node name;
|
||
|
|
yourself.
|
||
|
|
]
|