diff --git a/Dockerfile b/Dockerfile index f3e1ae9..d5ab149 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,3 +15,4 @@ LABEL databox.type="app" EXPOSE 8080 CMD ["npm","start"] +#CMD ["sleep","3000000"] diff --git a/README.md b/README.md index ba322ff..60fc7a8 100644 --- a/README.md +++ b/README.md @@ -26,14 +26,91 @@ This driver writes twitter event data into a store-json for later processing. It saves the following streams of data: - 1. twitterUserTimeLine - the logged in users timeline - 2. twitterHashTagStream - tweets that contain #raspberrypi, #mozfest, #databox, #iot and #NobelPrize + 1. twitterHashTagStream - tweets that contain #raspberrypi, #mozfest, #databox, #iot and #NobelPriz (hashtags can be changed in the driver settings) + +These can then be accessed store-json API. + +The driver used to provide the following from the user stream, but this +is no longer supported: + + 2. twitterUserTimeLine - the logged in users timeline 3. twitterDirectMessage - a list of the users' direct messages 4. twittrRetweet - a list of the users' retweets 5. twitterFavorite - a list of the users favourited -These can then be accessed store-json API. +The latter items can now only be accessed via webhooks, so not direct to +a NATed databox driver! + +## A tweet +Here's an example tweet (i.e. what appears as the data: value): +``` +{ created_at: 'Thu Oct 03 20:19:00 +0000 2019', + id: 1179853378215694300, + id_str: '1179853378215694336', + text: 'well, to be honest, quite of lot of time is trying to make databox work for #enablingtechnologies2019', + source: 'Twitter Web App', + truncated: false, + in_reply_to_status_id: null, + in_reply_to_status_id_str: null, + in_reply_to_user_id: null, + in_reply_to_user_id_str: null, + in_reply_to_screen_name: null, + user: + { id: 1176966706696245200, + id_str: '1176966706696245249', + name: 'Chris Greenhalgh', + screen_name: 'ChrisGreenhal14', + location: null, + url: null, + description: null, + translator_type: 'none', + protected: false, + verified: false, + followers_count: 0, + friends_count: 0, + listed_count: 0, + favourites_count: 0, + statuses_count: 4, + created_at: 'Wed Sep 25 21:08:45 +0000 2019', + utc_offset: null, + time_zone: null, + geo_enabled: false, + lang: null, + contributors_enabled: false, + is_translator: false, + profile_background_color: 'F5F8FA', + profile_background_image_url: '', + profile_background_image_url_https: '', + profile_background_tile: false, + profile_link_color: '1DA1F2', + profile_sidebar_border_color: 'C0DEED', + profile_sidebar_fill_color: 'DDEEF6', + profile_text_color: '333333', + profile_use_background_image: true, + profile_image_url: 'http://abs.twimg.com/sticky/default_profile_images/default_profile_normal.png', + profile_image_url_https: 'https://abs.twimg.com/sticky/default_profile_images/default_profile_normal.png', + default_profile: true, + default_profile_image: false, + following: null, + follow_request_sent: null, + notifications: null }, + geo: null, + coordinates: null, + place: null, + contributors: null, + is_quote_status: false, + quote_count: 0, + reply_count: 0, + retweet_count: 0, + favorite_count: 0, + entities: { hashtags: [Array], urls: [], user_mentions: [], symbols: [] }, + favorited: false, + retweeted: false, + filter_level: 'low', + lang: 'en', + timestamp_ms: '1570133940639' } +``` # Implementing OAuth in Databox diff --git a/databox-manifest.json b/databox-manifest.json index a97ab30..e16eb49 100644 --- a/databox-manifest.json +++ b/databox-manifest.json @@ -1,7 +1,7 @@ { "manifest-version": 1, "name": "driver-twitter", - "version": "0.3.1", + "version": "0.4.1", "description": "A simple Databox driver to stream twitter data", "author": "Anthony Brown ()", "license": "MIT", @@ -38,35 +38,15 @@ }, "provides": [ - { - "data-source-type": "twitterUserTimeLine", - "description": "Twitter user timeline data", - "store-type": "tsblob", - "schema": {} - }, { "data-source-type": "twitterHashTagStream", "description": "Twitter user hashtag data", - "store-type": "tsblob", - "schema": {} - }, - { - "data-source-type": "twitterDirectMessage", - "description": "Twitter users direct messages", - "store-type": "tsblob", - "schema": {} - }, - { - "data-source-type": "twitterFavourites", - "description": "Twitter users favourites tweets", - "store-type": "tsblob", - "schema": {} + "store-type": "ts/blob" }, { "data-source-type": "Test Actuator", "description": "testActuator", - "store-type": "tsblob", - "schema": {} + "store-type": "ts/blob" } ] } diff --git a/package.json b/package.json index e5a8418..fe3e46d 100644 --- a/package.json +++ b/package.json @@ -38,7 +38,7 @@ "body-parser": "^1.18.3", "express": "^4.16.3", "modclean": "^2.1.2", - "node-databox": "^0.9.0", + "node-databox": "^0.10.7", "oauth": "^0.9.15", "pug": "^2.0.3", "twit": "^2.2.10" diff --git a/src/main.js b/src/main.js index 6753937..7fc1643 100644 --- a/src/main.js +++ b/src/main.js @@ -4,6 +4,7 @@ const express = require("express"); const bodyParser = require("body-parser"); const oauth = require('oauth'); const databox = require('node-databox'); +const dns = require('dns'); const twitter = require('./twitter.js')(); @@ -14,9 +15,10 @@ try { DefaultTwitConfig = {}; } +const DATABOX_ARBITER_ENDPOINT = process.env.DATABOX_ARBITER_ENDPOINT || 'tcp://127.0.0.1:4444'; const DATABOX_ZMQ_ENDPOINT = process.env.DATABOX_ZMQ_ENDPOINT; -const credentials = databox.getHttpsCredentials(); +const credentials = databox.GetHttpsCredentials(); const PORT = process.env.port || '8080'; @@ -25,6 +27,7 @@ const HASH_TAGS_TO_TRACK = ['#raspberrypi', '#mozfest', '#databox', '#iot', '#No const app = express(); app.use(bodyParser.urlencoded({extended: true})); +app.use(bodyParser.json()); app.use('/ui', express.static('./src/www')); app.set('views', './src/views'); @@ -46,6 +49,7 @@ app.post('/ui/login', (req, res) => { getSettings() .then((settings) => { let urlRoot = req.body.callback; + console.log("[/ui/login] override callback = "+urlRoot); if (urlRoot == null) { urlRoot = "https://localhost/driver-twitter/ui/oauth" } @@ -90,7 +94,7 @@ app.get('/ui/oauth', (req, res) => { setSettings(settings) .then(() => { let redirectURL = settings.redirect; - redirectURL = redirectURL.replace('/driver-twitter/ui/oauth', '/core-ui/ui/view?ui=driver-twitter') + redirectURL = redirectURL.replace('/driver-twitter/ui/oauth', '/core-ui/ui/view/driver-twitter') twitter.connect(settings) .then( (T)=> { monitorTwitterEvents(T, settings); @@ -117,9 +121,9 @@ app.post('/ui/logout', (req, res) => { }); -app.get('/ui/setHashTags', function (req, res) { - let newHashTags = req.query.hashTags; - console.log(newHashTags); +app.post('/ui/setHashTags', function (req, res) { + console.log("[setHashTags] body", req.body); + let newHashTags = req.body.hashTags; getSettings() .then((settings) => { settings.hashTags = newHashTags.split(','); @@ -151,8 +155,7 @@ console.log("[Creating server]"); https.createServer(credentials, app).listen(PORT); module.exports = app; -let tsc = databox.NewTimeSeriesBlobClient(DATABOX_ZMQ_ENDPOINT, false); -let kvc = databox.NewKeyValueClient(DATABOX_ZMQ_ENDPOINT, false); +let sc = databox.NewStoreClient(DATABOX_ZMQ_ENDPOINT, DATABOX_ARBITER_ENDPOINT, false); let timeLine = databox.NewDataSourceMetadata(); timeLine.Description = 'Twitter user timeline data'; @@ -160,7 +163,7 @@ timeLine.ContentType = 'application/json'; timeLine.Vendor = 'Databox Inc.'; timeLine.DataSourceType = 'twitterUserTimeLine'; timeLine.DataSourceID = 'twitterUserTimeLine'; -timeLine.StoreType = 'ts'; +timeLine.StoreType = 'ts/blob'; let hashTag = databox.NewDataSourceMetadata(); hashTag.Description = 'Twitter user hashtag data'; @@ -168,7 +171,7 @@ hashTag.ContentType = 'application/json'; hashTag.Vendor = 'Databox Inc.'; hashTag.DataSourceType = 'twitterHashTagStream'; hashTag.DataSourceID = 'twitterHashTagStream'; -hashTag.StoreType = 'ts'; +hashTag.StoreType = 'ts/blob'; let userDM = databox.NewDataSourceMetadata(); userDM.Description = 'Twitter users direct messages'; @@ -176,7 +179,7 @@ userDM.ContentType = 'application/json'; userDM.Vendor = 'Databox Inc.'; userDM.DataSourceType = 'twitterDirectMessage'; userDM.DataSourceID = 'twitterDirectMessage'; -userDM.StoreType = 'ts'; +userDM.StoreType = 'ts/blob'; let userRetweet = databox.NewDataSourceMetadata(); userRetweet.Description = 'Twitter users retweets'; @@ -184,7 +187,7 @@ userRetweet.ContentType = 'application/json'; userRetweet.Vendor = 'Databox Inc.'; userRetweet.DataSourceType = 'twitterRetweet'; userRetweet.DataSourceID = 'twitterRetweet'; -userRetweet.StoreType = 'ts'; +userRetweet.StoreType = 'ts/blob'; let userFav = databox.NewDataSourceMetadata(); userFav.Description = 'Twitter users favourites tweets'; @@ -192,7 +195,7 @@ userFav.ContentType = 'application/json'; userFav.Vendor = 'Databox Inc.'; userFav.DataSourceType = 'twitterFavourites'; userFav.DataSourceID = 'twitterFavourites'; -userFav.StoreType = 'ts'; +userFav.StoreType = 'ts/blob'; let testActuator = databox.NewDataSourceMetadata(); testActuator.Description = 'Test Actuator'; @@ -200,7 +203,7 @@ testActuator.ContentType = 'application/json'; testActuator.Vendor = 'Databox Inc.'; testActuator.DataSourceType = 'testActuator'; testActuator.DataSourceID = 'testActuator'; -testActuator.StoreType = 'ts'; +testActuator.StoreType = 'ts/blob'; testActuator.IsActuator = true; let driverSettings = databox.NewDataSourceMetadata(); @@ -212,33 +215,51 @@ driverSettings.DataSourceID = 'twitterSettings'; driverSettings.StoreType = 'kv'; -tsc.RegisterDatasource(timeLine) +sc.RegisterDatasource(hashTag) + // TODO find some way to replace this old functionality with the activity api +/* .then(() => { - return tsc.RegisterDatasource(hashTag); + return sc.RegisterDatasource(timeLine); }) .then(() => { - return tsc.RegisterDatasource(userDM); + return sc.RegisterDatasource(userDM); }) .then(() => { - return tsc.RegisterDatasource(userRetweet); + return sc.RegisterDatasource(userRetweet); }) .then(() => { - return tsc.RegisterDatasource(userFav); + return sc.RegisterDatasource(userFav); }) +*/ .then(() => { - return tsc.RegisterDatasource(testActuator); + return sc.RegisterDatasource(testActuator); }) .then(() => { - return kvc.RegisterDatasource(driverSettings); + return sc.RegisterDatasource(driverSettings); }) .catch((err) => { console.log("Error registering data source:" + err); }) + .then(() => new Promise(function (resolve,reject) { + // ensure core-network permissions are in place + let lookup = function() { + dns.resolve('api.twitter.com', function(err, records) { + if (err) { + console.log("DNS lookup failed; retrying..."); + setTimeout(lookup, 1000); + return; + } + console.log("DNS ok (for twitter)"); + resolve(); + }) + } + lookup(); + })) .then(() => { let inlineSettings = DefaultTwitConfig; inlineSettings.hashTags = HASH_TAGS_TO_TRACK; - getSettings() + return getSettings() .then((settings) => { console.log("Twitter Auth"); if (settings.hasOwnProperty('consumer_key')) { @@ -261,10 +282,13 @@ tsc.RegisterDatasource(timeLine) }) .catch((err) => { console.log("[ERROR]", err); - let settings = {} - settings.access_token = null; - settings.access_token_secret = null; - setSettings(settings); + getSettings() + .then((settings) => { + settings.access_token = null; + settings.access_token_secret = null; + setSettings(settings); + + }) }); let streams = []; @@ -278,8 +302,16 @@ const monitorTwitterEvents = (twit, settings) => { HashtagStream.on('tweet', function (tweet) { save('twitterHashTagStream', tweet); }); + HashtagStream.on('error', function(err) { + console.log("Hashtag stream error", err); + }); - const UserStream = twit.stream('user', {stringify_friend_ids: true, with: 'followings', replies: 'all'}); + // TODO find some way to replace this old functionality using + // the activity API?! but that needs webhooks... + // https://github.com/ttezel/twit/issues/509 + // https://developer.twitter.com/en/docs/accounts-and-users/subscribe-account-activity/overview +/* + const UserStream = twit.stream('user', {stringify_friend_ids: true, 'with': 'followings', replies: 'all'}); streams.push(UserStream); UserStream.on('tweet', function (event) { save('twitterUserTimeLine', event); @@ -300,6 +332,10 @@ const monitorTwitterEvents = (twit, settings) => { UserStream.on('direct_message', function (event) { save('twitterDirectMessage', event); }); + UserStream.on('error', function(err) { + console.log("User stream error", err); + }); +*/ }; function stopAllStreams() { @@ -312,7 +348,7 @@ function stopAllStreams() { function getSettings() { datasourceid = 'twitterSettings'; return new Promise((resolve, reject) => { - kvc.Read(datasourceid, "settings") + sc.KV.Read(datasourceid, "settings") .then((settings) => { console.log("[getSettings] read response = ", settings); if (Object.keys(settings).length === 0) { @@ -338,7 +374,7 @@ function getSettings() { function setSettings(settings) { let datasourceid = 'twitterSettings'; return new Promise((resolve, reject) => { - kvc.Write(datasourceid, "settings", settings) + sc.KV.Write(datasourceid, "settings", settings) .then(() => { console.log('[setSettings] settings saved', settings); resolve(settings); @@ -351,13 +387,13 @@ function setSettings(settings) { } function save(datasourceid, data) { - console.log("Saving tweet::", data.text); + console.log(`Saving tweet to ${datasourceid}::`, data.text); json = {"data": data}; - tsc.Write(datasourceid, data) + sc.TSBlob.Write(datasourceid, data) .then((resp) => { console.log("Save got response ", resp); }) .catch((error) => { console.log("Error writing to store:", error); }); -} \ No newline at end of file +} diff --git a/src/views/index.pug b/src/views/index.pug index 0438802..31fa760 100644 --- a/src/views/index.pug +++ b/src/views/index.pug @@ -22,7 +22,7 @@ html(lang='en') div.mdc-line-ripple div.mdc-text-field(style="width:100%") - input#consumer_secret.mdc-text-field__input(type="text", name="consumer_secret" value=consumer_secret) + input#consumer_secret.mdc-text-field__input(type="password", name="consumer_secret" value=consumer_secret) label.mdc-floating-label(for="consumer_secret") | Consumer Secret div.mdc-line-ripple @@ -55,4 +55,4 @@ html(lang='en') div.mdc-card__actions(style="flex-direction: row-reverse") div.mdc-card__action-buttons - button.mdc-button#save_hashtags(type='button' value='save') Save \ No newline at end of file + button.mdc-button#save_hashtags(type='button' value='save') Save diff --git a/src/www/bundle.js b/src/www/bundle.js index 904902f..2cdb971 100644 --- a/src/www/bundle.js +++ b/src/www/bundle.js @@ -8,7 +8,15 @@ if(oauth) { document.getElementById('save_hashtags').addEventListener('click', function () { const hashTags = document.getElementById('hashTags').value; - fetch('ui/setHashTags', {credentials: "include", body: JSON.stringify({'hashTags': hashTags})}) + console.log('hashTags', hashTags) + fetch('setHashTags', { + method: "POST", + credentials: "include", + headers: { + 'Content-Type': 'application/json' + }, + body: JSON.stringify({'hashTags': hashTags}) + }) .then(function () { document.getElementById('hashtag_msg').innerText = "Saved!" })