Rebranding: Merge branch 'master' of https://code.tupale.co/Offray/Datanalitica

This commit is contained in:
ruidajo 2022-04-01 17:39:52 -05:00
commit f323c51d2a
168 changed files with 1013 additions and 2 deletions

View File

@ -9,6 +9,6 @@ baseline: spec
"self rssTools: spec." "self rssTools: spec."
"Packages" "Packages"
spec spec
package: 'Datanalitica' package: 'Socialmetrica'
with: [ spec requires: #('XMLParserHTML' "'RSSTools'") ] with: [ spec requires: #('XMLParserHTML' "'RSSTools'") ]
] ]

View File

@ -6,6 +6,6 @@
"pools" : [ ], "pools" : [ ],
"classvars" : [ ], "classvars" : [ ],
"instvars" : [ ], "instvars" : [ ],
"name" : "BaselineOfDatanalitica", "name" : "BaselineOfSocialmetrica",
"type" : "normal" "type" : "normal"
} }

View File

@ -0,0 +1,5 @@
{
"separateMethodMetaAndSource" : false,
"noMethodMetaData" : true,
"useCypressPropertiesFile" : true
}

View File

@ -0,0 +1,14 @@
baselines
baseline: spec
<baseline>
spec
for: #common
do: [
"Dependencies"
self xmlParserHTML: spec.
"self rssTools: spec."
"Packages"
spec
package: 'Socialmetrica'
with: [ spec requires: #('XMLParserHTML' "'RSSTools'") ]
]

View File

@ -0,0 +1,10 @@
baselines
rssTools: spec
Metacello new
repository: 'github://brackendev/RSSTools-Pharo:v1.0.1/src';
baseline: 'RSSTools';
onConflict: [ :ex | ex useLoaded ];
onUpgrade: [ :ex | ex useLoaded ];
onDowngrade: [ :ex | ex useLoaded ];
load.
spec baseline: 'RSSTools' with: [ spec repository: 'github://brackendev/RSSTools-Pharo:v1.0.1/src']

View File

@ -0,0 +1,11 @@
baselines
xmlParserHTML: spec
Metacello new
baseline: 'XMLParserHTML';
repository: 'github://pharo-contributions/XML-XMLParserHTML/src';
onConflict: [ :ex | ex useLoaded ];
onUpgrade: [ :ex | ex useLoaded ];
onDowngrade: [ :ex | ex useLoaded ];
onWarningLog;
load.
spec baseline: 'XMLParserHTML' with: [spec repository: 'github://pharo-contributions/XML-XMLParserHTML/src']

View File

@ -0,0 +1,11 @@
{
"commentStamp" : "",
"super" : "BaselineOf",
"category" : "BaselineOfSocialmetrica",
"classinstvars" : [ ],
"pools" : [ ],
"classvars" : [ ],
"instvars" : [ ],
"name" : "BaselineOfSocialmetrica",
"type" : "normal"
}

View File

@ -0,0 +1 @@
SystemOrganization addCategory: #BaselineOfSocialmetrica!

View File

@ -0,0 +1 @@
(name 'BaselineOfSocialmetrica')

View File

@ -0,0 +1 @@
{ }

View File

@ -0,0 +1,5 @@
{
"separateMethodMetaAndSource" : false,
"noMethodMetaData" : true,
"useCypressPropertiesFile" : true
}

View File

@ -0,0 +1,3 @@
accessing
addKeyword: keyword to: subtopic
(self subtopics at: subtopic) add: keyword.

View File

@ -0,0 +1,3 @@
accessing
addSubtopic: subtopicString
self subtopics at: subtopicString put: Set new.

View File

@ -0,0 +1,4 @@
accessing
language: isoLanguageCode
"isoLanguageCode follows the ISO 639-1 two letters convention"
language := isoLanguageCode

View File

@ -0,0 +1,3 @@
accessing
language
^ language

View File

@ -0,0 +1,3 @@
accessing
name: aString
name := aString

View File

@ -0,0 +1,3 @@
accessing
name
^ name

View File

@ -0,0 +1,5 @@
accessing
printOn: aStream
super printOn: aStream.
aStream
nextPutAll: '( ',self name, ' | ', self language, ' )'

View File

@ -0,0 +1,3 @@
accessing
subtopics: subtopicsNamesArray
subtopicsNamesArray do: [:each | self addSubtopic: each ]

View File

@ -0,0 +1,3 @@
accessing
subtopics
^ subtopics ifNil: [ subtopics := Dictionary new ]

View File

@ -0,0 +1,15 @@
{
"commentStamp" : "",
"super" : "Object",
"category" : "Socialmetrica",
"classinstvars" : [ ],
"pools" : [ ],
"classvars" : [ ],
"instvars" : [
"language",
"name",
"subtopics"
],
"name" : "DiscourseTopic",
"type" : "normal"
}

View File

@ -0,0 +1,6 @@
accessing
nitterProvider
"For a full list of Nitter providers, see:
https://github.com/zedeus/nitter/wiki/Instances"
^ 'https://nitter.42l.fr/'

View File

@ -0,0 +1,7 @@
accessing
asDictionary
^ { 'profile-card-avatar' -> self profileImageFile fullName.
'profile-card-fullname' -> self name .
'profile-card-username' -> self userName .
'profile-bio' -> self profileBio } asDictionary

View File

@ -0,0 +1,4 @@
accessing
config: aDictionary
config := aDictionary

View File

@ -0,0 +1,4 @@
accessing
config
^ config

View File

@ -0,0 +1,6 @@
accessing
createdAt
^ createdAt ifNil: [| joinDateString |
joinDateString := ((self documentTree xpath: '//div[@class="profile-joindate"]/span/@title') stringValue).
createdAt := (ZTimestampFormat fromString:'4:05 PM - 03 Feb 2001') parse: joinDateString.
]

View File

@ -0,0 +1,3 @@
accessing
description
^ description ifNil: [description := (self documentTree xpath: '//div[@class="profile-bio"]') stringValue]

View File

@ -0,0 +1,3 @@
operation
documentTree
^ XMLHTMLParser parse: self userNameLink asUrl retrieveContents

View File

@ -0,0 +1,4 @@
accessing
downloadProfileImage
self exportProfileImageOn: self folder / self userName, '.jpg'

View File

@ -0,0 +1,12 @@
accessing
exportProfileImageOn: fileReference
| file |
file := fileReference asFileReference.
file ensureDelete.
file exists ifFalse: [ file ensureCreateFile ].
file binaryWriteStreamDo: [ :stream |
stream nextPutAll: profileImageUrl retrieveContents ].
profileImageFile := file.
super class inform: 'Exported as: ', String cr, file fullName.
^ file

View File

@ -0,0 +1,9 @@
accessing
exportWithTemplate: mustacheFile On: folder
| mustacheDoc |
self exportProfileImageOn:folder / userName, '-profileImage.jpg'.
mustacheDoc := mustacheFile asMustacheTemplate value: self asDictionary.
MarkupFile
exportAsFileOn: (folder / self userName , 'tex')
containing: mustacheDoc

View File

@ -0,0 +1,4 @@
accessing
folder
^ self config at: 'folder'.

View File

@ -0,0 +1,10 @@
accessing
getMessages
| lastTweetsRaw lastTweets |
lastTweetsRaw := self rssFeed xmlDocument xpath: '//item'.
lastTweets := TweetsCollection new.
lastTweetsRaw do: [ :rssTweet |
lastTweets add: ((Tweet new fromNitterRssItem: rssTweet ))
].
^ lastTweets

View File

@ -0,0 +1,3 @@
accessing
id
^ id ifNil: [id := (self profileImageUrl segments select: [ :each | each asInteger class = LargePositiveInteger]) first.]

View File

@ -0,0 +1,3 @@
accessing
name
^ name ifNil: [ name := ((self rssFeed requiredItems title) splitOn: '/') first ]

View File

@ -0,0 +1,4 @@
accessing
profileBio
^ profileBio := (self documentTree xpath: '/html/body/div/div/div[2]/div[1]/div[2]/div[1]') stringValue

View File

@ -0,0 +1,4 @@
accessing
profileImageUrl
^ profileImageUrl ifNil: [
profileImageUrl := ((self rssFeed xmlDocument xpath: '//image/url') stringValue copyReplaceAll: '%2F' with: '/') asUrl ]

View File

@ -0,0 +1,12 @@
accessing
retrieveContents
self userName ifNil: [^ self].
^ self
id;
name;
description;
createdAt;
url;
profileImageUrl;
profileBio;
yourself.

View File

@ -0,0 +1,4 @@
accessing
rssFeed
^ RSSTools createRSSFeedFor: self userNameLink , '/rss'

View File

@ -0,0 +1,7 @@
accessing
url
^ url ifNil: [ | temp |
temp := ((self documentTree xpath: '//div[@class="profile-website"]') // 'a' @@ 'href') first.
temp ifNil: [ ^ url := nil ].
url := temp asUrl.
]

View File

@ -0,0 +1,3 @@
accessing
userName: userNameString
userName := userNameString.

View File

@ -0,0 +1,4 @@
accessing
userNameLink
^ self class nitterProvider, self userName

View File

@ -0,0 +1,13 @@
{
"commentStamp" : "",
"super" : "TwitterUser",
"category" : "Socialmetrica",
"classinstvars" : [ ],
"pools" : [ ],
"classvars" : [ ],
"instvars" : [
"config"
],
"name" : "NitterUser",
"type" : "normal"
}

View File

@ -0,0 +1,62 @@
accessing
asCardElement
| aModeLook anEditor textInfoPane buttonsPane |
aModeLook := BrEditorModeAptitude new
editableFocused: [ :aWidget | aWidget border: (BlBorder paint: BrGlamorousColors focusedEditorBorderColor width: 1) ];
editableUnfocused: [ :aWidget | aWidget border: (BlBorder paint: BrGlamorousColors editorBorderColor width: 1) ];
readOnly: [ :aWidget | aWidget border: BlBorder empty ].
anEditor := BrEditor new
aptitude: BrGlamorousRegularEditorAptitude new + aModeLook;
text: self text;
vFitContent.
textInfoPane := BrVerticalPane new
hMatchParent;
vFitContent;
margin: (BlInsets left: 20);
addChild: (BrLabel new
aptitude: BrGlamorousLabelAptitude;
text: '@' , self user userName , ' | ' , self created asString;
beSmallSize);
addChild: anEditor.
buttonsPane := BrHorizontalPane new
fitContent;
cellSpacing: 5;
addChildren: {
BrButton new
aptitude: BrGlamorousButtonWithLabelAptitude new;
label: 'Toggle subtopics';
action: [ anEditor beEditable ].
BrButton new
aptitude: BrGlamorousButtonWithLabelAptitude new;
label: 'Add subtopic keyword';
action: [ anEditor beReadOnlyWithSelection ].
BrButton new
aptitude: BrGlamorousButtonWithLabelAptitude new;
label: 'Details';
action: [ :e | e phlow spawnObject: self ].
BrButton new
aptitude: BrGlamorousButtonWithLabelAptitude new;
label: 'Web view';
action: [ self webView ].
}.
^ BrHorizontalPane new
padding: (BlInsets all: 15);
margin: (BlInsets all: 10);
cellSpacing: 5;
hMatchParent;
vFitContent;
addChildren: {
(self user profileImage asElement asScalableElement size: 64 @ 64).
BrVerticalPane new
cellSpacing: 5;
hMatchParent;
vFitContent;
addChildren: {
buttonsPane.
textInfoPane.
}
}

View File

@ -0,0 +1,3 @@
accessing
created
^ created

View File

@ -0,0 +1,9 @@
accessing
fromDictionary: aDictionary
created := (aDictionary at: 'created_at') asDateAndTime.
text := aDictionary at: 'text'.
id := aDictionary at: 'id'.
authorId := aDictionary at: 'author_id'.
user := aDictionary at: 'username' ifAbsent: [''] .
conversationId := aDictionary at: 'conversation_id' ifAbsent: [ '' ].
^ self

View File

@ -0,0 +1,9 @@
accessing
fromNitter: aDictionary
created := (aDictionary at: 'pubDate') "asDateAndTime".
text := aDictionary at: 'text'.
"id := aDictionary at: 'id'.
authorId := aDictionary at: 'author_id'."
user := aDictionary at: 'creator'.
"conversationId := aDictionary at: 'conversation_id' ifAbsent: [ '' ]."
^ self

View File

@ -0,0 +1,9 @@
accessing
fromNitterRssItem: xmlItem
| author |
author := (xmlItem xpath: 'dc:creator') stringValue allButFirst.
user := NitterUser new
userName: author .
created := (xmlItem xpath: 'pubDate') stringValue.
text := (XMLHTMLParser on: (xmlItem xpath: 'description') stringValue) parseDocument stringValue.
id := ((xmlItem xpath: 'guid') stringValue splitOn: '/') last copyReplaceAll: '#m' with: ''

View File

@ -0,0 +1,16 @@
accessing
gtViewTweetDetailsOn: aView
<gtView>
^ aView explicit
title: 'Tweet Details' translated;
priority: 5;
stencil: [
BlElement new
layout: BlFlowLayout new;
constraintsDo: [ :c |
c vertical fitContent.
c horizontal matchParent ];
padding: (BlInsets all: 10);
addChild: (self asCardElement margin: (BlInsets all: 20))
]

View File

@ -0,0 +1,3 @@
accessing
id
^ id

View File

@ -0,0 +1,3 @@
queries
mentions: aWord
^ self text includesSubstring: aWord

View File

@ -0,0 +1,5 @@
accessing
printOn: aStream
super printOn: aStream.
aStream
nextPutAll: '( ',self text ,' )'

View File

@ -0,0 +1,3 @@
accessing
text
^ text

View File

@ -0,0 +1,3 @@
accessing
user: aTwitterUser
user := aTwitterUser

View File

@ -0,0 +1,3 @@
accessing
user
^ user

View File

@ -0,0 +1,3 @@
accessing
webView
WebBrowser openOn: 'https://twitter.com/', self user userName, '/status/', self id

View File

@ -0,0 +1,3 @@
utilities
words
^ self text allRegexMatches: '\w*'

View File

@ -0,0 +1,3 @@
accessing
wordsInLowercase
^ self words collect: [:word | word asLowercase ]

View File

@ -0,0 +1,18 @@
{
"commentStamp" : "",
"super" : "Object",
"category" : "Socialmetrica",
"classinstvars" : [ ],
"pools" : [ ],
"classvars" : [ ],
"instvars" : [
"created",
"text",
"id",
"authorId",
"conversationId",
"user"
],
"name" : "Tweet",
"type" : "normal"
}

View File

@ -0,0 +1,3 @@
accessing
add: aTweet
self tweets add: aTweet

View File

@ -0,0 +1,25 @@
ui
gtTweetsFor: aView
<gtView>
^ aView explicit
title: 'Tweets';
stencil: [
| container imageContainer |
container := BlElement new
layout: BlFlowLayout new;
constraintsDo: [ :c |
c vertical fitContent.
c horizontal matchParent ];
padding: (BlInsets all: 10).
self tweets do: [ :each |
imageContainer := BlLazyElement new
withGlamorousPreview;
aptitude: BrShadowAptitude new;
background: Color white;
margin: (BlInsets all: 10);
constraintsDo: [ :c |
c vertical exact: 145.
c horizontal matchParent ];
elementBuilder: [ each asCardElement margin: (BlInsets all: 20) ].
container addChild: imageContainer].
container asScrollableElement ]

View File

@ -0,0 +1,5 @@
accessing
printOn: aStream
super printOn: aStream.
aStream
nextPutAll: '( ',self size asString, ' Tweet(s) )'

View File

@ -0,0 +1,3 @@
accessing
size
^ self tweets size

View File

@ -0,0 +1,3 @@
accessing
tweets: aTweetsCollection
^ tweets := aTweetsCollection

View File

@ -0,0 +1,3 @@
accessing
tweets
^ tweets ifNil: [ tweets := OrderedCollection new]

View File

@ -0,0 +1,13 @@
{
"commentStamp" : "",
"super" : "Object",
"category" : "Socialmetrica",
"classinstvars" : [ ],
"pools" : [ ],
"classvars" : [ ],
"instvars" : [
"tweets"
],
"name" : "TweetsCollection",
"type" : "normal"
}

View File

@ -0,0 +1,3 @@
I model some parts of the Twitter API version 2 as described in:
<https://developer.twitter.com/en/docs/twitter-api/early-access>

View File

@ -0,0 +1,3 @@
accessing
apiKeysFile: aFileReference
apiKeysFile := aFileReference

View File

@ -0,0 +1,5 @@
accessing
apiKeysFile
"Return the defined apiKeysFile or assign a default location following the Linux Standard
File Hierarchy, which is relatively portable to other Operative Systems."
^ apiKeysFile ifNil: [ apiKeysFile := FileLocator home / '.config/Datanalitica/twitter-api-keys.json' ]

View File

@ -0,0 +1,3 @@
accessing
keys
^ keys ifNil: [ keys := Dictionary new]

View File

@ -0,0 +1,4 @@
accessing
loadKeys
keys := STONJSON fromString: self apiKeysFile contents.
^ keys

View File

@ -0,0 +1,3 @@
accessing
bearerToken
^ self class keys at: 'Bearer Token'

View File

@ -0,0 +1,7 @@
accessing
defaultQueryParameters
^ Dictionary new
at: 'tweetsOrig' put: '?tweet.fields=created_at&expansions=author_id&user.fields=created_at&max_results=100';
at: 'tweets' put: '?', 'tweet.fields=created_at', '&', 'expansions=author_id', '&', 'max_results=100';
at: 'mentionsOrig' put: '?expansions=author_id&tweet.fields=conversation_id,created_at,lang&user.fields=created_at,entities&max_results=100';
yourself

View File

@ -0,0 +1,3 @@
accessing
keys
^ keys ifNil: [ keys := self class loadKeys]

View File

@ -0,0 +1,4 @@
accessing
loadKeys
keys := self class loadKeys.
^ self

View File

@ -0,0 +1,3 @@
accessing
options: aDictionary
options := aDictionary

View File

@ -0,0 +1,9 @@
accessing
options
"Return the configuration options or define a default if they are not given"
^ options ifNil: [
options := Dictionary new
at: 'caching' put: true;
at: 'pagesPerRequest' put: '1';
yourself
]

View File

@ -0,0 +1,6 @@
accessing
rawResponseForURL: anUrl
^ ZnClient new
headerAt: 'Authorization' put: 'Bearer ', self bearerToken;
url: anUrl;
get.

View File

@ -0,0 +1,4 @@
accessing
storage: aFolder
storage := aFolder

View File

@ -0,0 +1,4 @@
accessing
storage
^ storage

View File

@ -0,0 +1,8 @@
accessing
userEndPointFor: username selecting: tweetsOrMentions
"I build a shared URL for querying last 100 mentions or tweets for a particular user.
Second parameter should be only 'tweets' or 'mentions', dateString, if present, should be YYYY-MM-DD."
| commonQueryParameters userFields |
userFields := 'user.fields=username,name,description,profile_image_url,created_at'.
commonQueryParameters := '?expansions=author_id&tweet.fields=conversation_id,created_at&', userFields, '&max_results=100'.
^ self usersBaseEndPoint, (self userIDFrom: username), '/', tweetsOrMentions, commonQueryParameters

View File

@ -0,0 +1,8 @@
as yet unclassified
userEndPointFor: username selecting: tweetsOrMentions since: dateString
"I build a shared URL for querying last 100 mentions or tweets for a particular user.
Second parameter should be only 'tweets' or 'mentions', dateString should be YYYY-MM-DD."
| commonQueryParameters |
commonQueryParameters := '?expansions=author_id&tweet.fields=conversation_id,created_at&user.fields=username&max_results=100',
'&start_time=', dateString,'T00:00:00Z&'.
^ self usersBaseEndPoint, (self userIDFrom: username), '/', tweetsOrMentions, commonQueryParameters

View File

@ -0,0 +1,5 @@
queries
userIDFrom: username
| rawResponse |
rawResponse := self rawResponseForURL: self usersBaseEndPoint, 'by/username/', username.
^ (STONJSON fromString: rawResponse) at: 'data' at: 'id'

View File

@ -0,0 +1,4 @@
accessing
userMentionsFor: username
"The following query gets the last 100 mentions that is the maximun allowed for a particular user without pagination:"
^ self userQueryFor: username selecting: 'mentions'

View File

@ -0,0 +1,18 @@
accessing
userMentionsFor: username since: startDateString
| nextToken queryUrl sinceDate untilDate messages response |
sinceDate := 'start_time=',startDateString, 'T00:00:00Z'.
messages := OrderedCollection new.
nextToken := ''.
[ nextToken includesSubstring: 'stop' ] whileFalse: [
queryUrl := self usersBaseEndPoint,
(self userIDFrom: username), '/mentions', (self defaultQueryParameters at: 'mentionsOrig') ,
'&', sinceDate,
'&', nextToken.
response := STONJSON fromString: (self rawResponseForURL: queryUrl).
(response at: 'data') do: [:tweetData |
messages add: (Tweet new fromDictionary: tweetData)
].
nextToken := 'pagination_token=',((response at: 'meta') at: 'next_token' ifAbsent: [ 'stop' ])].
^ messages.

View File

@ -0,0 +1,21 @@
accessing
userMentionsFor: username since: startDateString until: endDateString
| nextToken queryUrl sinceDate untilDate messages response extraQueryParamenters |
sinceDate := 'start_time=',startDateString, 'T17:00:00Z'.
untilDate := 'end_time=',endDateString, 'T01:00:00Z'.
extraQueryParamenters := '?expansions=author_id&tweet.fields=conversation_id&user.fields=created_at,entities&max_results=100'.
messages := OrderedCollection new.
nextToken := ''.
[ nextToken includesSubstring: 'stop' ] whileFalse: [
queryUrl := self usersBaseEndPoint,
(self userIDFrom: username), '/mentions', extraQueryParamenters,
'&', sinceDate,
"'&', untilDate,"
'&', nextToken.
response := STONJSON fromString: (self rawResponseForURL: queryUrl).
(response at: 'data') do: [:tweetData |
messages add: (Tweet new fromDictionary: tweetData)
].
nextToken := 'pagination_token=',((response at: 'meta') at: 'next_token' ifAbsent: [ 'stop' ])].
^ messages.

View File

@ -0,0 +1,10 @@
accessing
userQueryFor: username selecting: tweetsOrMentions
| rawResponse queryURL |
"The following query gets the last 100 tweets or mentions that is the maximun allowed for a particular user without pagination:"
queryURL := self userEndPointFor: username selecting: tweetsOrMentions.
rawResponse := self rawResponseForURL:queryURL.
^ TwitterAPIResponse new
fromDictionary: (STONJSON fromString: rawResponse);
queryURL: queryURL;
date: DateAndTime now.

View File

@ -0,0 +1,4 @@
accessing
userTweetsFrom: username
"The following query gets the last 100 tweets, that is the maximun allowed for a particular user without pagination:"
^ self userQueryFor: username selecting: 'tweets'

View File

@ -0,0 +1,20 @@
accessing
userTweetsFrom: username since: startDateString until: endDateString
| nextToken queryUrl sinceDate untilDate messages response |
sinceDate := 'start_time=',startDateString, 'T00:00:00Z'.
untilDate := 'end_time=',endDateString, 'T23:59:59Z'.
messages := OrderedCollection new.
nextToken := ''.
[ nextToken includesSubstring: 'stop' ] whileFalse: [
queryUrl := self usersBaseEndPoint,
(self userIDFrom: username), '/tweets', (self defaultQueryParameters at: 'tweets'),
'&', sinceDate,
'&', untilDate,
'&', nextToken.
response := STONJSON fromString: (self rawResponseForURL: queryUrl).
(response at: 'data') do: [:tweetData |
messages add: (Tweet new fromDictionary: tweetData)
].
nextToken := 'pagination_token=',((response at: 'meta') at: 'next_token' ifAbsent: [ 'stop' ])].
^ messages.

View File

@ -0,0 +1,3 @@
utilities api
usersBaseEndPoint
^ 'https://api.twitter.com/2/users/'

View File

@ -0,0 +1,10 @@
accessing
usersGroupMentioning: userName
| response |
response := self userQueryFor: userName selecting: 'mentions'.
^ TwitterUsersGroup new
users: response messagesAuthors;
title: 'Users mentioning @', userName;
origin: response queryURL;
date: DateAndTime now;
storage: self storage.

Some files were not shown because too many files have changed in this diff Show More