✨A general (slight) cleanup

This should get things working again, at least on the unsafe branch!

examples/main.go → cmd/main.go

@@ -1,7 +1,7 @@
 package main
 
 import (
-	"discordtidal"
+	"github.com/unickorn/discordtidal"
 )
 
 func main() {

discord/assets.go

@@ -1,26 +1,27 @@
 package discord
 
 import (
-	"discordtidal/log"
-	"discordtidal/tidal"
 	"encoding/base64"
 	"encoding/json"
 	"fmt"
 	"github.com/df-mc/goleveldb/leveldb"
+	"github.com/unickorn/discordtidal/log"
+	"github.com/unickorn/discordtidal/tidal"
 	"net/http"
 	"sort"
 	"strconv"
+	"strings"
 	"time"
 )
 
-// Asset is an image object
+// Asset is an image object.
 type Asset struct {
 	// Name is the "Asset key" of the uploaded Asset. We use album ids here.
 	Name string `json:"name"`
 	// Type seems to be 1 most (all) of the time, I'm not sure what it is.
 	Type int `json:"type"`
-	// Id is the identifier of the uploaded Asset, an 18 characters long string.
-	Id string `json:"id"`
+	// ID is the identifier of the uploaded Asset, an 18 characters long string.
+	ID string `json:"id"`
 }
 
 // AssetData is a struct holding an Asset and its data.
@@ -54,18 +55,17 @@ func OpenDb() {
 	}
 	db = i
 
-	log.Log().Info("opened database")
+	log.Log().Infoln("Opened database")
 }
 
 // Sync loads the assets from Discord and synchronizes the data with the database.
 // The assets that are on Discord get marked or newly saved as "OK".
 // If there are more than 250 assets in total, oldest 10 assets are deleted.
 func Sync() {
-	log.Log().Info("syncing discord")
+	log.Log().Infoln("Syncing Discord cache")
 
-	t := 0
-	// get all of the uploaded ones, set them as OK
-	r, err := http.Get(fmt.Sprintf("https://discord.com/api/v9/oauth2/applications/%s/assets", config.ApplicationId))
+	// get all the uploaded assets
+	r, err := http.Get(fmt.Sprintf(endpointAssets, config.ApplicationId))
 	if err != nil {
 		panic(err)
 	}
@@ -75,24 +75,29 @@ func Sync() {
 		panic(err)
 	}
 
+	// set existing assets as OK on leveldb
 	existing := make(map[string]byte)
 	for _, a := range all {
-		t++
-		existing[a.Id] = 1
+		existing[a.ID] = 1
+
+		// check whether on leveldb
 		d, err := db.Get([]byte(a.Name), nil)
 		if d == nil {
-			err = db.Put([]byte(a.Name), hash(a.Id, assetOK, 0), nil)
+			// exists on discord but not on leveldb, so add it and set OK
+			err = db.Put([]byte(a.Name), hash(a.ID, assetOK, 0), nil)
 		} else {
+			// exists on both, so set OK while keeping the last opened time
 			_, _, lastOpened := unhash(d)
-			err = db.Put([]byte(a.Name), hash(a.Id, assetOK, lastOpened), nil)
+			log.Log().Debugf("[UPDATE] asset %s exists on both, setting OK!", a.Name)
+			err = db.Put([]byte(a.Name), hash(a.ID, assetOK, lastOpened), nil)
 		}
 		if err != nil {
 			panic(err)
 		}
 	}
 
-	shouldDelete := t > 250
-	log.Log().Infof("loaded %d assets", t)
+	shouldDelete := len(all) > 250
+	log.Log().Infof("Loaded %d assets", len(all))
 
 	var allFromDb []temp
 
@@ -110,8 +115,8 @@ func Sync() {
 			}
 		}
 
-		// if it's needed, throw the Asset in a slice so we can sort and delete oldest ones
-		// this is to make sure we don't surpass the 300 Asset limit
+		// if needed, throw the asset in a slice, so we can sort and delete the oldest ones
+		// this is to make sure we don't surpass the 300 asset limit
 		if shouldDelete && status == assetOK {
 			allFromDb = append(allFromDb, temp{
 				albumid:    string(iter.Key()),
@@ -121,45 +126,47 @@ func Sync() {
 		}
 	}
 	iter.Release()
+	err = iter.Error()
+	if err != nil {
+		panic(err)
+	}
 
 	// sort all assets and delete the oldest ones
 	if shouldDelete {
-		log.Log().Info("deleting stuff")
+		log.Log().Infoln("Deleting older assets to stay away from discord's asset limit")
+		// sort by last opened
 		sort.Slice(allFromDb, func(i, j int) bool {
 			return allFromDb[i].lastopened > allFromDb[j].lastopened
 		})
 
 		go func() {
-			for n := 0; n < 10; n++ { // only delete 10 so we don't get rate limited !!!!
-				// send delete to discord!!!!
+			for n := 0; n < 10; n++ { // only delete 10 and sleep in between, so we don't get rate limited !
+				// send delete to discord
 				a := allFromDb[n]
 				DoDeleteAsset(a.assetid)
 
-				// delete from db too !!!!!
+				// mark db as deleted
 				err = db.Put([]byte(a.albumid), hash(a.assetid, assetDeleted, a.lastopened), nil)
 				if err != nil {
 					panic(err)
 				}
-
-				time.Sleep(time.Second * 2)
+				time.Sleep(time.Second)
 			}
 		}()
 	}
-
-	err = iter.Error()
-	if err != nil {
-		panic(err)
-	}
 }
 
 // FetchAsset returns an AssetData only if the album is uploaded and OK, otherwise uploads it.
 func FetchAsset(album tidal.Album) *AssetData {
 	ad := GetAsset(album.StringId())
 	if ad != nil {
+		log.Log().Debugln("[FETCH] asset exists on db")
 		if ad.Status == assetOK {
+			log.Log().Debugln("-- [FETCH] asset OK")
 			return ad
 		}
 		if ad.Status == assetNew {
+			log.Log().Debugln("-- [FETCH] asset NEW")
 			return nil
 		}
 	}
@@ -179,32 +186,32 @@ func GetAsset(albumId string) *AssetData {
 		Asset: Asset{
 			Name: albumId,
 			Type: 1, // !
-			Id:   assetId,
+			ID:   assetId,
 		},
 		LastOpened: lastOpened,
 	}
 }
 
-// SaveAsset uploads an album cover to Discord and adds it to the database with the Status "new".
+// SaveAsset uploads an album cover to Discord and adds it to the database with the assetNew status.
 func SaveAsset(album tidal.Album) {
-	log.Log().Infof("asset for album %s not found, uploading new...", album.Title)
-	a := DoUploadAsset(album.StringId(), "data:image/jpg;base64,"+base64.StdEncoding.EncodeToString(album.GetCover()))
-	err := db.Put([]byte(album.StringId()), hash(a.Id, assetNew, int(time.Now().Unix())), nil)
+	log.Log().Infof("Asset for album %s not found, uploading new...", album.Title)
+	a := DoUploadAsset(album.StringId(), "data:image/jpg;base64,"+base64.StdEncoding.EncodeToString(album.CoverImage()))
+	err := db.Put([]byte(album.StringId()), hash(a.ID, assetNew, int(time.Now().Unix())), nil)
 	if err != nil {
 		panic(err)
 	}
 }
 
-// this is bad but it's my first time using leveldb ok
-// plus it doesn't really matter we don't need perfect performance
+// hash stores the asset id, status and last opened time into a byte array.
 func hash(id string, status AssetStatus, lastOpened int) []byte {
-	return []byte(id + strconv.Itoa(int(status)) + strconv.Itoa(lastOpened))
+	return []byte(fmt.Sprintf("%s:%d:%d", id, status, lastOpened))
 }
 
 // asset id, asset status, last opened
 func unhash(h []byte) (string, AssetStatus, int) {
 	str := string(h)
-	s, _ := strconv.Atoi(str[18:19])
-	l, _ := strconv.Atoi(str[19 : len(h)-1])
-	return str[0:18], AssetStatus(s), l
+	split := strings.Split(str, ":")
+	st, _ := strconv.ParseInt(split[1], 10, 8)
+	lastOpen, _ := strconv.Atoi(split[2])
+	return split[0], AssetStatus(st), lastOpen
 }

discord/config.go

@@ -0,0 +1,49 @@
+package discord
+
+import (
+	"github.com/pelletier/go-toml"
+	"github.com/unickorn/discordtidal/log"
+	"io/ioutil"
+)
+
+var config *Config
+
+// Config is a struct that holds the configuration for discordtidal.
+type Config struct {
+	ApplicationId string
+	Token         string
+	UserAgent     string
+	LogLevel      string
+}
+
+// LoadConfig loads the config.
+func LoadConfig() {
+	c := Config{}
+
+	b, err := ioutil.ReadFile("config.toml")
+	if err != nil {
+		log.Log().Infoln("Config not found, creating new")
+		data, err := toml.Marshal(c)
+		if err != nil {
+			panic(err)
+		}
+		if err := ioutil.WriteFile("config.toml", data, 0644); err != nil {
+			panic(err)
+		}
+		return
+	}
+	if err := toml.Unmarshal(b, &c); err != nil {
+		panic(err)
+	}
+	if c.LogLevel == "" {
+		c.LogLevel = "info"
+	}
+	log.SetLevel(c.LogLevel)
+	log.Log().Infoln("Config loaded!")
+	config = &c
+}
+
+// GetConfig returns the config.
+func GetConfig() *Config {
+	return config
+}

discord/delete_asset.go

@@ -1,16 +1,15 @@
 package discord
 
 import (
-	"discordtidal/log"
 	"fmt"
+	"github.com/unickorn/discordtidal/log"
 	"net/http"
 )
 
 // DeleteAsset is the interaction to delete an Asset from the Discord application.
 type DeleteAsset struct {
 	// id is the identifier of the Asset to be deleted.
 	id string
-	EndpointInterface
 }
 
 // DoDeleteAsset deletes a given Asset using an id.
@@ -24,7 +23,7 @@ func DoDeleteAsset(id string) {
 }
 
 func (d *DeleteAsset) url() string {
-	return fmt.Sprintf(deleteAsset, config.ApplicationId, d.id)
+	return fmt.Sprintf(endpointDeleteAsset, config.ApplicationId, d.id)
 }
 
 func (d *DeleteAsset) method() string {

discord/endpoint.go

@@ -3,6 +3,7 @@ package discord
 import (
 	"bytes"
 	"encoding/json"
+	"io"
 	"net/http"
 )
 
@@ -15,22 +16,18 @@ type EndpointInterface interface {
 
 // Do sends an EndpointInterface to Discord with the appropriate headers and returns the response.
 func Do(e EndpointInterface, hasBody bool) (*http.Response, error) {
-	var req *http.Request
-	var err error
-
+	var body io.Reader
 	if hasBody {
 		j, err := json.Marshal(e)
 		if err != nil {
 			panic(err)
 		}
-		req, err = http.NewRequest(e.method(), e.url(), bytes.NewReader(j))
-	} else {
-		req, err = http.NewRequest(e.method(), e.url(), nil)
+		body = bytes.NewReader(j)
 	}
+	req, err := http.NewRequest(e.method(), e.url(), body)
 	if err != nil {
 		panic(err)
 	}
-
 	req.Header.Set("Host", "discord.com")
 	req.Header.Set("Accept", "*/*")
 	req.Header.Set("Accept-Encoding", "gzip, deflate, br")
@@ -39,7 +36,6 @@ func Do(e EndpointInterface, hasBody bool) (*http.Response, error) {
 	req.Header.Set("Connection", "keep-alive")
 	req.Header.Set("Content-Type", "application/json")
 	req.Header.Set("User-Agent", config.UserAgent)
-
 	r, err := http.DefaultClient.Do(req)
 	return r, err
 }

discord/endpoints.go

@@ -1,50 +1,7 @@
 package discord
 
-import (
-	"discordtidal/log"
-	"github.com/pelletier/go-toml"
-	"io/ioutil"
-)
-
 const (
-	editApplicationData = "https://discord.com/api/v9/applications/%s"
-	assets              = "https://discord.com/api/v9/oauth2/applications/%s/assets"
-	deleteAsset         = "https://discord.com/api/v9/oauth2/applications/%s/assets/%s"
+	endpointEditApplicationData = "https://discord.com/api/v9/applications/%s"
+	endpointAssets              = "https://discord.com/api/v9/oauth2/applications/%s/assets"
+	endpointDeleteAsset         = "https://discord.com/api/v9/oauth2/applications/%s/assets/%s"
 )
-
-type Config struct {
-	ApplicationId string
-	Token         string
-	UserAgent     string
-}
-
-var config Config
-
-func LoadConfig() {
-	config = Config{
-		ApplicationId: "",
-		Token:         "",
-		UserAgent:     "",
-	}
-
-	b, err := ioutil.ReadFile("config.toml")
-	if err != nil {
-		log.Log().Info("config not found, creating new")
-		data, err := toml.Marshal(config)
-		if err != nil {
-			panic(err)
-		}
-		if err := ioutil.WriteFile("config.toml", data, 0644); err != nil {
-			panic(err)
-		}
-		return
-	}
-	if err := toml.Unmarshal(b, &config); err != nil {
-		panic(err)
-	}
-	log.Log().Infof("config loaded!")
-}
-
-func GetConfig() *Config {
-	return &config
-}