From b198bde247af78757bc3d8d21ac24a4abf023875 Mon Sep 17 00:00:00 2001
From: ruidajo <diegoruidajo@gmail.com>
Date: Thu, 7 Apr 2022 14:17:17 -0500
Subject: [PATCH] Improving  urls for load more tweets and removing unnecessary
 method. ref  5b169c4ebf

---
 .../instance/defaultConfig.st                 |  5 ---
 .../instance/numberOfURLsForLoadingTweets..st | 41 +++++++++++++------
 .../instance/fromNitterTimelineItem..st       | 12 ++++++
 3 files changed, 41 insertions(+), 17 deletions(-)
 delete mode 100644 Socialmetrica.package/NitterUser.class/instance/defaultConfig.st
 create mode 100644 Socialmetrica.package/Tweet.class/instance/fromNitterTimelineItem..st

diff --git a/Socialmetrica.package/NitterUser.class/instance/defaultConfig.st b/Socialmetrica.package/NitterUser.class/instance/defaultConfig.st
deleted file mode 100644
index 89e6d1b..0000000
--- a/Socialmetrica.package/NitterUser.class/instance/defaultConfig.st
+++ /dev/null
@@ -1,5 +0,0 @@
-accessing
-defaultConfig
-
-	self config: { 'folder' -> (FileLocator userData / 'Socialmetrica' / self userName) } asDictionary.
-	^ self config
\ No newline at end of file
diff --git a/Socialmetrica.package/NitterUser.class/instance/numberOfURLsForLoadingTweets..st b/Socialmetrica.package/NitterUser.class/instance/numberOfURLsForLoadingTweets..st
index 04f9cef..8d6063b 100644
--- a/Socialmetrica.package/NitterUser.class/instance/numberOfURLsForLoadingTweets..st
+++ b/Socialmetrica.package/NitterUser.class/instance/numberOfURLsForLoadingTweets..st
@@ -1,18 +1,35 @@
 accessing
 numberOfURLsForLoadingTweets: number
 
-	| collectionURLs count asURLs |
-	collectionURLs := {
-	self userNameLink .
-	(self userNameLink, ((self documentTree xPath: '//a[.="Load more"]') @ 'href') stringValue) .} asOrderedCollection.
+	| collectionURLs count asURLs urlAndTweets |
+	number = 1 ifTrue: [ ^ self ].
 	
-	number <= 2 ifTrue: [ ^ collectionURLs ].
-	count := 2.
-	(number-count) timesRepeat: [ | tempDoc | 
-		tempDoc := XMLHTMLParser parse: (collectionURLs at: count) asUrl retrieveContents.
-	collectionURLs 
-		add: (self userNameLink, 
-			((tempDoc xPath: '//a[.="Load more"]') @ 'href') stringValue).
+	urlAndTweets := OrderedDictionary new.
+	collectionURLs := { self userNameLink .}asOrderedCollection.
+	
+	urlAndTweets 
+			at: 'tweets'
+			put: ((self documentTree xpath: '//div[@class="timeline-item "]')asOrderedCollection collect: [ :xmlElement | xmlElement postCopy]).
+	
+	count := 1.
+	(number-count) timesRepeat: [ | tempDoc docTree urlString | 
+		tempDoc := XMLHTMLParser parse: 
+						(collectionURLs at: count) asUrl retrieveContents.
+			
+		urlString := (self userNameLink, 
+				((tempDoc xPath: '//a[.="Load more"]') @ 'href') stringValue).
+		docTree := XMLHTMLParser parse: urlString asUrl retrieveContents.
+		collectionURLs 
+			add: urlString.
+		
+		urlAndTweets 
+			at: 'tweets-', (urlString splitOn: 'cursor=') second
+			put: ((docTree xpath: '//div[@class="timeline-item "]')asOrderedCollection 
+					collect: [ :xmlElement | xmlElement postCopy ]).
 		count := count+1 ].
+		
 	asURLs := collectionURLs collect: [ :string | string asUrl ].
-	^ asURLs.
\ No newline at end of file
+	
+	urlAndTweets at: 'urls' put: asURLs.
+	
+	^ urlAndTweets.
\ No newline at end of file
diff --git a/Socialmetrica.package/Tweet.class/instance/fromNitterTimelineItem..st b/Socialmetrica.package/Tweet.class/instance/fromNitterTimelineItem..st
new file mode 100644
index 0000000..84df717
--- /dev/null
+++ b/Socialmetrica.package/Tweet.class/instance/fromNitterTimelineItem..st
@@ -0,0 +1,12 @@
+accessing
+fromNitterTimelineItem: xmlItem
+	| author authorAndId |
+	authorAndId := (((xmlItem xpath: '//a[@class="tweet-link"]') 
+					asString splitOn: 'href="' ) second splitOn: '/') reject: [ :i | i isEmpty or: [i = '>)']].
+	author := authorAndId first.
+	user := NitterUser new
+		userName: author .
+	"created := (xmlItem xpath: 'pubDate') stringValue.
+	text := (xmlItem xpath: 'description') stringValue."
+	id := authorAndId last copyReplaceAll: '#m"' with: ''.
+	authorId := self user id.
\ No newline at end of file