"
I am capable of creating PPCTokenizingParserNode.

This is a node that 
- has all all the tokens collected, 
- identifies the whitespace 
- replaces all the token reads with token consume
"
Class {
	#name : 'PPCTokenizingVisitor',
	#superclass : 'PPCRewritingVisitor',
	#instVars : [
		'tokens'
	],
	#category : 'PetitCompiler-Visitors'
}

{ #category : 'tokens' }
PPCTokenizingVisitor >> addToken: token [
	(tokens contains: [:e | e = token] ) ifFalse: [ 
		tokens addLast: token
	]
]

{ #category : 'hooks' }
PPCTokenizingVisitor >> afterAccept: node retval: retval [
	| newRetval |
	self isRoot ifTrue: [ 
		| tokensNode whitespaceNode |

		tokens do: [ :token | token unmarkForInline ].
		whitespaceNode := tokens detect: [ :e | e isTrimmingTokenNode ] ifNone: [ nil ].
		whitespaceNode notNil ifTrue: [ 
				whitespaceNode := whitespaceNode whitespace copy
					unmarkForInline;
					name: 'consumeWhitespace';
					yourself.
				"whitespaceNode := PPCTokenWhitespaceNode new
					child: whitespaceNode;
					yourself"
		] ifFalse: [ 
				whitespaceNode := PPCNilNode new
					name: 'consumeWhitespace';
					yourself 
		].
		
		tokensNode := PPCListNode new
			children: tokens asArray;
			name: 'nextToken';
			yourself.
			

		 newRetval := PPCTokenizingParserNode new
			parser: retval;
			whitespace: whitespaceNode;
			tokens: tokensNode;
			name: (retval name isNil ifTrue: [ #mainParser ] ifFalse: [ retval name ]);
			yourself 
	] ifFalse: [ 
		newRetval := retval
	].

	^ super afterAccept: node retval: newRetval
	
	"Modified: / 12-05-2015 / 01:37:57 / Jan Vrany <jan.vrany@fit.cvut.cz>"
]

{ #category : 'hooks' }
PPCTokenizingVisitor >> eofToken [
	| ws  |
	self error: 'deprecated?'.
	ws := PPCStarNode new
		child: (PPCMessagePredicateNode new
			message: #isSeparator;
			yourself);
		yourself.
	
	^ PPCTrimmingTokenNode new
		child: PPCEndOfFileNode new;
		whitespace: ws;
		tokenClass: PPToken;
		name: 'eof';
		yourself.
]

{ #category : 'initialization' }
PPCTokenizingVisitor >> initialize [
	super initialize.
	tokens := OrderedCollection new.
]

{ #category : 'testing' }
PPCTokenizingVisitor >> isRoot [
	^ openSet size = 1
]

{ #category : 'visiting' }
PPCTokenizingVisitor >> visitTokenConsumeNode: node [
	"
		Seems, it might happen, that if I create the consume node, 
		I will ge to it later. This would create a token consume node for the
		child, thus having tokenConsumNode with tokenConsumNode as a child...
	"
	^ node
]

{ #category : 'visiting' }
PPCTokenizingVisitor >> visitTokenNode: node [
	self addToken: node.
	
	self assert: node acceptsEpsilon not description: 'Sorry, but the epsilon tokens are not allowed'.
	
	^ PPCTokenConsumeNode new
		child: node;
		name: node name;
		yourself.
]

{ #category : 'visiting' }
PPCTokenizingVisitor >> visitTrimmingTokenNode: node [
	self addToken: node.
	
	^ PPCTokenConsumeNode new
		child: node;
		name: node name;
		yourself.
]