504 lines
15 KiB
Smalltalk
504 lines
15 KiB
Smalltalk
"
|
|
I'm Twitt
|
|
|
|
I store twitts from twitter (http://twitter.com).
|
|
|
|
"
|
|
Class {
|
|
#name : #Tweet,
|
|
#superclass : #Object,
|
|
#instVars : [
|
|
'date',
|
|
'message',
|
|
'profile',
|
|
'url',
|
|
'mentions',
|
|
'links',
|
|
'hashtags',
|
|
'type',
|
|
'retweetedUser',
|
|
'repliedUsers'
|
|
],
|
|
#category : #'Dataviz-Twitter'
|
|
}
|
|
|
|
{ #category : #accessing }
|
|
Tweet >> date [
|
|
^ date
|
|
]
|
|
|
|
{ #category : #accessing }
|
|
Tweet >> date: anObject [
|
|
date := anObject
|
|
]
|
|
|
|
{ #category : #'data scrapping' }
|
|
Tweet >> detectMessageTypeFrom: aJSONSnippet [
|
|
"Given aJSONSnippet containing a Tweet message data, I detect the type of message inside,
|
|
between tweet, retweet or reply to conver it to a native Tweet object"
|
|
(aJSONSnippet keys includes: 'in_reply_to_status_id')
|
|
ifTrue: [
|
|
self
|
|
type: 'reply';
|
|
url: '/', (self profile, '/status/', (aJSONSnippet at: 'id_str')) ]
|
|
ifFalse: [(aJSONSnippet keys includes: 'retweeted_status')
|
|
ifTrue: [
|
|
self
|
|
type: 'retweet';
|
|
url: '/',
|
|
(((aJSONSnippet at: 'retweeted_status') at: 'user') at: 'screen_name'),
|
|
'/status/',
|
|
((aJSONSnippet at: 'retweeted_status') at: 'id_str') ]
|
|
ifFalse: [
|
|
self
|
|
type: 'tweet';
|
|
url: '/', (self profile, '/status/', (aJSONSnippet at: 'id_str')) ] ].
|
|
]
|
|
|
|
{ #category : #'data scrapping' }
|
|
Tweet >> detectRepliedProfilesFrom: aJSONSnippet [
|
|
"Given aJSONSnippet containing a retweet message data, I detect the profiles that were mentioned."
|
|
self repliedUsers: (aJSONSnippet at: 'in_reply_to_screen_name' ifAbsent: [ self repliedUsers: nil ])
|
|
"| initalMentions |
|
|
initalMentions := (aJSONSnippet at: 'entities') at: 'user_mentions'.
|
|
initalMentions isEmpty
|
|
ifFalse: [ self mentions: nil ]
|
|
ifTrue: [
|
|
) ]."
|
|
]
|
|
|
|
{ #category : #'data scrapping' }
|
|
Tweet >> detectRetweetedProfileFrom: aJSONSnippet [
|
|
"Given aJSONSnippet containing a retweet message data, I detect the profile that was retweeted."
|
|
(aJSONSnippet keys includes: 'retweeted_status')
|
|
ifFalse: [ self retweetedUser: nil ]
|
|
ifTrue: [
|
|
self retweetedUser: (((aJSONSnippet at: 'retweeted_status') at: 'user') at: 'screen_name') ].
|
|
]
|
|
|
|
{ #category : #accessing }
|
|
Tweet >> hashtags [
|
|
^ hashtags
|
|
]
|
|
|
|
{ #category : #accessing }
|
|
Tweet >> hashtags: anObject [
|
|
hashtags := anObject
|
|
]
|
|
|
|
{ #category : #'data scrapping' }
|
|
Tweet >> impact [
|
|
"Computes the maximum amount of followers that a tweet can reach, including the retweets.
|
|
It doesn't take into account that retweeters can share followers and in future versions
|
|
this should be computed, even if we don't have a full list of followers.
|
|
Also the computing time could be sustancially improved by extracting only followers data
|
|
from the profiles instead of scrapping all and by having a better cache which includes
|
|
prescrapped data.
|
|
'Real time' is not the main issue here, but 'slow data'.
|
|
For example we could include snapshots of the retweeters profiles and to aks for a
|
|
commented retweet to reach new followers"
|
|
|
|
| retweetsHtml retweetsTree retweeters impactData |
|
|
|
|
retweetsHtml := ZnClient new get: 'http://mutabit.com/deltas/repos.fossil/datapolis/doc/tip/Data/Sources/Single/', ((self url splitOn: '//') at: 2), '/retweets.html'.
|
|
|
|
retweetsTree := Soup fromString: retweetsHtml.
|
|
|
|
retweeters := (retweetsTree findAllTagsByClass: 'username')
|
|
collect: [:each | each text copyReplaceAll: '@' with: ''].
|
|
|
|
impactData := Dictionary new.
|
|
|
|
impactData
|
|
at: 'retweeters' put: retweeters size;
|
|
at: 'reach' put: (retweeters collect: [:each | (TwitterProfile new scrapDataForProfile: each) followers]) sum.
|
|
|
|
^ impactData.
|
|
]
|
|
|
|
{ #category : #'data scrapping' }
|
|
Tweet >> impactFor: aTweetUrl [
|
|
"Computes the maximum amount of followers that a tweet can reach, including the retweets.
|
|
It doesn't take into account that retweeters can share followers and in future versions
|
|
this should be computed, even if we don't have a full list of followers.
|
|
Also the computing time could be sustancially improved by extracting only followers data
|
|
from the profiles instead of scrapping all and by having a better cache which includes
|
|
prescrapped data.
|
|
'Real time' is not the main issue here, but 'slow data'.
|
|
For example we could include snapshots of the retweeters profiles and to aks for a
|
|
commented retweet to reach new followers"
|
|
|
|
| retweetsHtml retweetsTree retweeters impactData |
|
|
|
|
retweetsHtml := ZnClient new get: 'http://mutabit.com/deltas/repos.fossil/datapolis/doc/tip/Data/Sources/Single/', ((aTweetUrl splitOn: '//') at: 2), '/retweets.html'.
|
|
|
|
retweetsTree := Soup fromString: retweetsHtml.
|
|
|
|
retweeters := (retweetsTree findAllTagsByClass: 'username')
|
|
collect: [:each | each text copyReplaceAll: '@' with: ''].
|
|
|
|
impactData := Dictionary new.
|
|
|
|
impactData
|
|
at: 'retweeters' put: retweeters size;
|
|
at: 'reach' put: (retweeters collect: [:each | (TwitterProfile new scrapFollowersForProfile: each)]) sum.
|
|
|
|
^ impactData.
|
|
]
|
|
|
|
{ #category : #initialization }
|
|
Tweet >> initialize [
|
|
"Creates a new TwitterProfile object"
|
|
super initialize.
|
|
]
|
|
|
|
{ #category : #'data visualization' }
|
|
Tweet >> labeledCircleSized: diameter upperLabel: label1 bottomLabel: label2 inView: v [
|
|
|
|
| circle ex1 ex2 line label1Temp label2Temp radious origin background els |
|
|
|
|
background := (RTBox new color: Color transparent) element.
|
|
v add: background.
|
|
|
|
origin := (0@0).
|
|
radious := diameter / 2.
|
|
|
|
circle := (RTEllipse new size: diameter"; borderColor: Color black; color: Color transparent") element.
|
|
ex1 := RTBox element.
|
|
ex2 := RTBox element.
|
|
circle translateTo: origin.
|
|
ex1 translateTo: ((radious negated @ 0) + circle center).
|
|
ex2 translateTo: (radious @ 0) + circle center.
|
|
line := (RTLine new; color: Color black) edgeFrom: ex1 to: ex2.
|
|
v add: circle; add: line.
|
|
|
|
label1Temp := (RTLabel text: label1) element.
|
|
label2Temp := (RTLabel text: label2) element.
|
|
v add: label1Temp ; add: label2Temp .
|
|
label1Temp translateTo: (0 @ -12) + circle center.
|
|
label2Temp translateTo: (0 @ 12) + circle center.
|
|
|
|
els := RTGroup with: label1Temp with: label2Temp with: circle with: ex1 with: ex2.
|
|
RTNest new on: background nest: els.
|
|
|
|
^ background
|
|
]
|
|
|
|
{ #category : #accessing }
|
|
Tweet >> links [
|
|
^ links
|
|
]
|
|
|
|
{ #category : #accessing }
|
|
Tweet >> links: anObject [
|
|
links := anObject
|
|
]
|
|
|
|
{ #category : #accessing }
|
|
Tweet >> mentions [
|
|
^ mentions
|
|
]
|
|
|
|
{ #category : #accessing }
|
|
Tweet >> mentions: anObject [
|
|
mentions := anObject
|
|
]
|
|
|
|
{ #category : #'data visualization' }
|
|
Tweet >> mentionsClusterSeparated: aDistance inView: aView [
|
|
"I return a graphic of all avatars mentioned in a tweet, layered as a cluster a separated
|
|
by aDistance"
|
|
|
|
| container avatars profiles |
|
|
|
|
"Profiles: Create profiles collecting information for all users mentioned in the tweet"
|
|
profiles := OrderedCollection new.
|
|
self mentions do: [:eachMention |
|
|
profiles add: (TwitterProfile new scrapDataForProfile: eachMention)].
|
|
|
|
"Avatars: Use the profiles to create a collection of avatar images which are draggable
|
|
elements on a view"
|
|
avatars := OrderedCollection new.
|
|
profiles do: [:eachProfile |
|
|
avatars add: (RTBitmap new form: eachProfile avatar) element ].
|
|
RTCircleLayout new
|
|
initialRadius: (avatars size)*aDistance;
|
|
on: avatars.
|
|
|
|
container := (RTBox new color: Color transparent) element .
|
|
RTNest new on: container nest: avatars.
|
|
|
|
"Adding avatars and its labels"
|
|
aView addAll: avatars.
|
|
aView add: container.
|
|
|
|
1 to: (avatars size) do: [:i |
|
|
(avatars at: i) @ (RTLabelled new
|
|
text: (('@',(self mentions at: i)));
|
|
fontSize: 35;
|
|
color: (Color black))].
|
|
^ container
|
|
]
|
|
|
|
{ #category : #accessing }
|
|
Tweet >> message [
|
|
^ message
|
|
]
|
|
|
|
{ #category : #accessing }
|
|
Tweet >> message: anObject [
|
|
message := anObject
|
|
]
|
|
|
|
{ #category : #accessing }
|
|
Tweet >> profile [
|
|
^ profile
|
|
]
|
|
|
|
{ #category : #accessing }
|
|
Tweet >> profile: anObject [
|
|
profile := anObject
|
|
]
|
|
|
|
{ #category : #accessing }
|
|
Tweet >> repliedUsers [
|
|
^ repliedUsers
|
|
]
|
|
|
|
{ #category : #accessing }
|
|
Tweet >> repliedUsers: anObject [
|
|
repliedUsers := anObject
|
|
]
|
|
|
|
{ #category : #accessing }
|
|
Tweet >> retweetedUser [
|
|
^ retweetedUser
|
|
]
|
|
|
|
{ #category : #accessing }
|
|
Tweet >> retweetedUser: anObject [
|
|
retweetedUser := anObject
|
|
]
|
|
|
|
{ #category : #'data scrapping' }
|
|
Tweet >> scrapDataFromUrl: aTweetUrl [
|
|
"Scraps most of the data in the page of a aTweetUrl. Most of the tweets are prestored now, but in the future
|
|
it will combine prestored data and live streaming data"
|
|
|
|
| tweetSourceHtml tweetSourceTree hour dateTmp day month year|
|
|
tweetSourceHtml := ZnClient new get:
|
|
'http://mutabit.com/deltas/repos.fossil/datapolis/doc/tip/Data/Sources/Single/',
|
|
((aTweetUrl splitOn: '//') at: 2), '/tweet.html'.
|
|
|
|
tweetSourceTree := Soup fromString: tweetSourceHtml.
|
|
self url: aTweetUrl .
|
|
self profile: ((aTweetUrl splitOn: '/') at: 4).
|
|
self message: ((((tweetSourceTree findAllTagsByClass: 'tweet') at: 1) children at: 6) children at: 2).
|
|
self mentions: ((self message findAllTagsByClass: 'twitter-atreply')
|
|
collect: [:each | (each attributeAt: 'href') copyReplaceAll: '/' with: '']).
|
|
self hashtags: ((self message findAllTagsByClass: 'twitter-hashtag') collect: [:each | each text]).
|
|
self links: ((self message findAllTagsByClass: 'twitter-timeline-link')
|
|
collect: [:each | each attributeAt: 'data-url']).
|
|
hour := ((((tweetSourceTree findAllTagsByClass: 'tweet') at: 1) children at: 8) children at: 2).
|
|
dateTmp := (((((tweetSourceTree findAllTagsByClass: 'tweet') at: 1) children at: 8) children at: 4) text) splitOn: ' '.
|
|
day := dateTmp at: 3.
|
|
month := dateTmp at: 4.
|
|
year := '20', (dateTmp at: 5).
|
|
self date: (year, month, day, hour text, hour nextSibling text) asDateAndTime.
|
|
]
|
|
|
|
{ #category : #'data visualization' }
|
|
Tweet >> showInView: aView [
|
|
"shows the general information (author, author's avatar and text) of the tweet in a view"
|
|
|
|
| tweetPanel tweetText holder avatar |
|
|
|
|
"Tweet text"
|
|
(self message text size > 50)
|
|
ifTrue:[
|
|
tweetText := RTLabel new
|
|
text: (self splitByLines at: 1), String cr,
|
|
(self splitByLines at: 2), String cr, String cr,
|
|
(self date asString)]
|
|
ifFalse: [tweetText := RTLabel new
|
|
text: (self splitByLines at: 1), String cr, String cr,
|
|
(self date asString) ].
|
|
tweetText height: 35.
|
|
tweetText := tweetText element.
|
|
|
|
"Avatar"
|
|
avatar := (RTBitmap new form:
|
|
((TwitterProfile new scrapDataForProfile: self profile) avatar)) element.
|
|
avatar translateBy: (-30*(30)@0).
|
|
|
|
"Putting tweet elements together"
|
|
tweetPanel := OrderedCollection new.
|
|
tweetPanel add: avatar; add: tweetText.
|
|
holder := (RTBox new color: Color transparent) element @ RTDraggable.
|
|
RTNest new on: holder nest: tweetPanel.
|
|
|
|
"Adding elements and labels to the view"
|
|
aView add: tweetText.
|
|
aView add: avatar.
|
|
aView add: holder.
|
|
avatar @ (RTLabelled new
|
|
text: '@', (self profile);
|
|
fontSize: 35;
|
|
color: Color black;
|
|
top;
|
|
offsetOnEdge: 1 ).
|
|
^ holder.
|
|
]
|
|
|
|
{ #category : #'data visualization' }
|
|
Tweet >> showInView: aView sized: aSize [
|
|
"shows the general information (author, author's avatar and text) of the tweet in a view"
|
|
|
|
| tweetPanel tweetText holder avatar |
|
|
|
|
"Tweet text"
|
|
(self message text size > 50)
|
|
ifTrue:[
|
|
tweetText := RTLabel new
|
|
text: (self splitByLines at: 1), String cr,
|
|
(self splitByLines at: 2), String cr, String cr,
|
|
(self date asString)]
|
|
ifFalse: [tweetText := RTLabel new
|
|
text: (self splitByLines at: 1), String cr, String cr,
|
|
(self date asString) ].
|
|
tweetText height: aSize.
|
|
tweetText := tweetText element.
|
|
|
|
"Avatar"
|
|
avatar := (RTBitmap new form:
|
|
((TwitterProfile new scrapDataForProfile: self profile) avatar)) element.
|
|
avatar translateBy: (-30*aSize@0).
|
|
|
|
"Putting tweet elements together"
|
|
tweetPanel := OrderedCollection new.
|
|
tweetPanel add: avatar; add: tweetText.
|
|
holder := (RTBox new color: Color transparent) element @ RTDraggable.
|
|
RTNest new on: holder nest: tweetPanel.
|
|
|
|
"Adding elements and labels to the view"
|
|
aView add: tweetText.
|
|
aView add: avatar.
|
|
aView add: holder.
|
|
avatar @ (RTLabelled new
|
|
text: '@', (self profile);
|
|
fontSize: 35;
|
|
color: Color black;
|
|
top;
|
|
offsetOnEdge: 1 ).
|
|
^ holder.
|
|
]
|
|
|
|
{ #category : #'data visualization' }
|
|
Tweet >> silenceMapFor: aTweetUrl [
|
|
|
|
"Creates a visualization of how long a tweet has not been answered (any kind of answer: not favs,
|
|
not RT, no nothing)"
|
|
|
|
| v tweet impact line timeUnanswered mentionedAvatars dummyData lineLabelUp lineLabelDown |
|
|
|
|
self scrapDataFromUrl: aTweetUrl.
|
|
timeUnanswered := (Date today - self date) days.
|
|
|
|
dummyData := self impactFor: aTweetUrl.
|
|
|
|
"Impact box"
|
|
impact := RTBox new
|
|
color: (Color red);
|
|
size: 200.
|
|
|
|
impact := impact element .
|
|
impact translateBy: (timeUnanswered*(-50))@0.
|
|
|
|
"Adding objects to the view, except avatars"
|
|
v := RTView new.
|
|
mentionedAvatars := self mentionsClusterSeparated: 70 inView: v.
|
|
tweet := self showInView: v sized: 35.
|
|
tweet translateBy: (timeUnanswered*(-25))@(-600).
|
|
v add: impact.
|
|
"Line"
|
|
line := RTEdge from: mentionedAvatars to: impact.
|
|
v add: (line + (RTGradientColoredLine new
|
|
colors: (Array with: (Color white alpha:0.3) with: (Color red alpha:0.9));
|
|
precision: 100;
|
|
width: 20;
|
|
gradientColorShape)).
|
|
|
|
"Adding line labels"
|
|
lineLabelUp := (RTLabel new
|
|
text: timeUnanswered asString, ' días sin respuesta';
|
|
height: timeUnanswered * 2.5) element @ RTDraggable.
|
|
lineLabelUp translateBy: (timeUnanswered*(-25))@(-105).
|
|
|
|
lineLabelDown := (RTLabel new
|
|
text: 'al ', Date today asString, ' y contando...';
|
|
height: timeUnanswered * 1;
|
|
color: Color gray) element @ RTDraggable.
|
|
lineLabelDown translateBy: (timeUnanswered*(-15))@(50).
|
|
|
|
v add: lineLabelUp; add: lineLabelDown.
|
|
|
|
|
|
impact @ (RTLabelled new
|
|
text: (((dummyData at: 'retweeters') asString), ' retweets');
|
|
fontSize: timeUnanswered * 2;
|
|
color: Color gray).
|
|
|
|
impact @ (RTLabelled new
|
|
text: (((dummyData at: 'reach') asString), ' lectores', String cr, '(max)');
|
|
fontSize: timeUnanswered * 2;
|
|
color: Color gray;
|
|
below).
|
|
|
|
"Showing the canvas"
|
|
|
|
v view canvas focusOnCenterScaled.
|
|
^ v @ RTDraggableView.
|
|
|
|
]
|
|
|
|
{ #category : #'as yet unclassified' }
|
|
Tweet >> splitByLines [
|
|
"Splits the message text of the tweet in lines, in case of being too long for a better rendering
|
|
in silenceMap and other views."
|
|
|
|
| summa i charLimit words wordSizes lines |
|
|
|
|
words := self message text splitOn: ' '.
|
|
wordSizes := words collect: [:eachWord | eachWord size].
|
|
i := 1.
|
|
summa := 0.
|
|
charLimit := 50.
|
|
[summa < charLimit] whileTrue: [
|
|
summa := (wordSizes at: i) + summa.
|
|
i := i +1].
|
|
lines := OrderedCollection new.
|
|
lines add: (Character space join: (words copyFrom: 1 to: i)).
|
|
lines add: (Character space join: (words copyFrom: i + 1 to: words size)).
|
|
^ lines
|
|
]
|
|
|
|
{ #category : #accessing }
|
|
Tweet >> type [
|
|
^ type
|
|
]
|
|
|
|
{ #category : #accessing }
|
|
Tweet >> type: anObject [
|
|
type := anObject
|
|
]
|
|
|
|
{ #category : #accessing }
|
|
Tweet >> url [
|
|
^ url
|
|
]
|
|
|
|
{ #category : #accessing }
|
|
Tweet >> url: anObject [
|
|
url := anObject
|
|
]
|