Skip to content

Commit

Permalink
Merge pull request #3772 from advplyr/feed-episodes-upsert
Browse files Browse the repository at this point in the history 
Feed episode IDs changing on refresh & several other refresh issues
  • Loading branch information
@advplyr
advplyr authored Jan 1, 2025
2 parents 8c4d0c5 + e7f7d1a commit c8a0592
Show file tree
Hide file tree
Showing 3 changed files with 80 additions and 30 deletions.
31 changes: 23 additions & 8 deletions server/managers/RssFeedManager.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -98,11 +98,22 @@ class RssFeedManager {
podcastId: feed.entity.mediaId
},
attributes: ['id', 'updatedAt'],
order: [['createdAt', 'DESC']]
order: [['updatedAt', 'DESC']]
})

if (mostRecentPodcastEpisode && mostRecentPodcastEpisode.updatedAt > newEntityUpdatedAt) {
newEntityUpdatedAt = mostRecentPodcastEpisode.updatedAt
}
} else {
const book = await Database.bookModel.findOne({
where: {
id: feed.entity.mediaId
},
attributes: ['id', 'updatedAt']
})
if (book && book.updatedAt > newEntityUpdatedAt) {
newEntityUpdatedAt = book.updatedAt
}
}

return newEntityUpdatedAt > feed.entityUpdatedAt
Expand All @@ -111,7 +122,7 @@ class RssFeedManager {
attributes: ['id', 'updatedAt'],
include: {
model: Database.bookModel,
attributes: ['id'],
attributes: ['id', 'audioFiles', 'updatedAt'],
through: {
attributes: []
},
Expand All @@ -122,13 +133,16 @@ class RssFeedManager {
}
})

const totalBookTracks = feed.entity.books.reduce((total, book) => total + book.includedAudioFiles.length, 0)
if (feed.feedEpisodes.length !== totalBookTracks) {
return true
}

let newEntityUpdatedAt = feed.entity.updatedAt

const mostRecentItemUpdatedAt = feed.entity.books.reduce((mostRecent, book) => {
if (book.libraryItem.updatedAt > mostRecent) {
return book.libraryItem.updatedAt
}
return mostRecent
let updatedAt = book.libraryItem.updatedAt > book.updatedAt ? book.libraryItem.updatedAt : book.updatedAt
return updatedAt > mostRecent ? updatedAt : mostRecent
}, 0)

if (mostRecentItemUpdatedAt > newEntityUpdatedAt) {
Expand All @@ -151,6 +165,9 @@ class RssFeedManager {
let feed = await Database.feedModel.findOne({
where: {
slug: req.params.slug
},
include: {
model: Database.feedEpisodeModel
}
})
if (!feed) {
Expand All @@ -163,8 +180,6 @@ class RssFeedManager {
if (feedRequiresUpdate) {
Logger.info(`[RssFeedManager] Feed "${feed.title}" requires update - updating feed`)
feed = await feed.updateFeedForEntity()
} else {
feed.feedEpisodes = await feed.getFeedEpisodes()
}

const xml = feed.buildXml(req.originalHostPrefix)
Expand Down
30 changes: 21 additions & 9 deletions server/models/Feed.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,9 @@ class Feed extends Model {
entityUpdatedAt = libraryItem.media.podcastEpisodes.reduce((mostRecent, episode) => {
return episode.updatedAt > mostRecent ? episode.updatedAt : mostRecent
}, entityUpdatedAt)
} else if (libraryItem.media.updatedAt > entityUpdatedAt) {
// Book feeds will use Book.updatedAt if more recent
entityUpdatedAt = libraryItem.media.updatedAt
}

const feedObj = {
Expand Down Expand Up @@ -187,7 +190,8 @@ class Feed extends Model {
const booksWithTracks = collectionExpanded.books.filter((book) => book.includedAudioFiles.length)

const entityUpdatedAt = booksWithTracks.reduce((mostRecent, book) => {
return book.libraryItem.updatedAt > mostRecent ? book.libraryItem.updatedAt : mostRecent
const updatedAt = book.libraryItem.updatedAt > book.updatedAt ? book.libraryItem.updatedAt : book.updatedAt
return updatedAt > mostRecent ? updatedAt : mostRecent
}, collectionExpanded.updatedAt)

const firstBookWithCover = booksWithTracks.find((book) => book.coverPath)
Expand Down Expand Up @@ -275,7 +279,8 @@ class Feed extends Model {
static getFeedObjForSeries(userId, seriesExpanded, slug, serverAddress, feedOptions = null) {
const booksWithTracks = seriesExpanded.books.filter((book) => book.includedAudioFiles.length)
const entityUpdatedAt = booksWithTracks.reduce((mostRecent, book) => {
return book.libraryItem.updatedAt > mostRecent ? book.libraryItem.updatedAt : mostRecent
const updatedAt = book.libraryItem.updatedAt > book.updatedAt ? book.libraryItem.updatedAt : book.updatedAt
return updatedAt > mostRecent ? updatedAt : mostRecent
}, seriesExpanded.updatedAt)

const firstBookWithCover = booksWithTracks.find((book) => book.coverPath)
Expand Down Expand Up @@ -516,17 +521,24 @@ class Feed extends Model {
try {
const updatedFeed = await this.update(feedObj, { transaction })

// Remove existing feed episodes
await feedEpisodeModel.destroy({
where: {
feedId: this.id
},
transaction
})
const existingFeedEpisodeIds = this.feedEpisodes.map((ep) => ep.id)

// Create new feed episodes
updatedFeed.feedEpisodes = await feedEpisodeCreateFunc(feedEpisodeCreateFuncEntity, updatedFeed, this.slug, transaction)

const newFeedEpisodeIds = updatedFeed.feedEpisodes.map((ep) => ep.id)
const feedEpisodeIdsToRemove = existingFeedEpisodeIds.filter((epid) => !newFeedEpisodeIds.includes(epid))

if (feedEpisodeIdsToRemove.length) {
Logger.info(`[Feed] Removing ${feedEpisodeIdsToRemove.length} episodes from feed ${this.id}`)
await feedEpisodeModel.destroy({
where: {
id: feedEpisodeIdsToRemove
},
transaction
})
}

await transaction.commit()

return updatedFeed
Expand Down
49 changes: 36 additions & 13 deletions server/models/FeedEpisode.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,9 +53,10 @@ class FeedEpisode extends Model {
* @param {import('./Feed')} feed
* @param {string} slug
* @param {import('./PodcastEpisode')} episode
* @param {string} [existingEpisodeId]
*/
static getFeedEpisodeObjFromPodcastEpisode(libraryItemExpanded, feed, slug, episode) {
const episodeId = uuidv4()
static getFeedEpisodeObjFromPodcastEpisode(libraryItemExpanded, feed, slug, episode, existingEpisodeId = null) {
const episodeId = existingEpisodeId || uuidv4()
return {
id: episodeId,
title: episode.title,
Expand Down Expand Up @@ -94,11 +95,18 @@ class FeedEpisode extends Model {
libraryItemExpanded.media.podcastEpisodes.sort((a, b) => new Date(a.pubDate) - new Date(b.pubDate))
}

let numExisting = 0
for (const episode of libraryItemExpanded.media.podcastEpisodes) {
feedEpisodeObjs.push(this.getFeedEpisodeObjFromPodcastEpisode(libraryItemExpanded, feed, slug, episode))
// Check for existing episode by filepath
const existingEpisode = feed.feedEpisodes?.find((feedEpisode) => {
return feedEpisode.filePath === episode.audioFile.metadata.path
})
numExisting = existingEpisode ? numExisting + 1 : numExisting

feedEpisodeObjs.push(this.getFeedEpisodeObjFromPodcastEpisode(libraryItemExpanded, feed, slug, episode, existingEpisode?.id))
}
Logger.info(`[FeedEpisode] Creating ${feedEpisodeObjs.length} episodes for feed ${feed.id}`)
return this.bulkCreate(feedEpisodeObjs, { transaction })
Logger.info(`[FeedEpisode] Upserting ${feedEpisodeObjs.length} episodes for feed ${feed.id} (${numExisting} existing)`)
return this.bulkCreate(feedEpisodeObjs, { transaction, updateOnDuplicate: ['title', 'author', 'description', 'siteURL', 'enclosureURL', 'enclosureType', 'enclosureSize', 'pubDate', 'season', 'episode', 'episodeType', 'duration', 'filePath', 'explicit'] })
}

/**
Expand Down Expand Up @@ -127,11 +135,12 @@ class FeedEpisode extends Model {
* @param {string} slug
* @param {import('./Book').AudioFileObject} audioTrack
* @param {boolean} useChapterTitles
* @param {string} [existingEpisodeId]
*/
static getFeedEpisodeObjFromAudiobookTrack(book, pubDateStart, feed, slug, audioTrack, useChapterTitles) {
static getFeedEpisodeObjFromAudiobookTrack(book, pubDateStart, feed, slug, audioTrack, useChapterTitles, existingEpisodeId = null) {
// Example: <pubDate>Fri, 04 Feb 2015 00:00:00 GMT</pubDate>
let timeOffset = isNaN(audioTrack.index) ? 0 : Number(audioTrack.index) * 1000 // Offset pubdate to ensure correct order
let episodeId = uuidv4()
let episodeId = existingEpisodeId || uuidv4()

// e.g. Track 1 will have a pub date before Track 2
const audiobookPubDate = date.format(new Date(pubDateStart.valueOf() + timeOffset), 'ddd, DD MMM YYYY HH:mm:ss [GMT]')
Expand Down Expand Up @@ -179,11 +188,18 @@ class FeedEpisode extends Model {
const useChapterTitles = this.checkUseChapterTitlesForEpisodes(libraryItemExpanded.media)

const feedEpisodeObjs = []
let numExisting = 0
for (const track of libraryItemExpanded.media.trackList) {
feedEpisodeObjs.push(this.getFeedEpisodeObjFromAudiobookTrack(libraryItemExpanded.media, libraryItemExpanded.createdAt, feed, slug, track, useChapterTitles))
// Check for existing episode by filepath
const existingEpisode = feed.feedEpisodes?.find((episode) => {
return episode.filePath === track.metadata.path
})
numExisting = existingEpisode ? numExisting + 1 : numExisting

feedEpisodeObjs.push(this.getFeedEpisodeObjFromAudiobookTrack(libraryItemExpanded.media, libraryItemExpanded.createdAt, feed, slug, track, useChapterTitles, existingEpisode?.id))
}
Logger.info(`[FeedEpisode] Creating ${feedEpisodeObjs.length} episodes for feed ${feed.id}`)
return this.bulkCreate(feedEpisodeObjs, { transaction })
Logger.info(`[FeedEpisode] Upserting ${feedEpisodeObjs.length} episodes for feed ${feed.id} (${numExisting} existing)`)
return this.bulkCreate(feedEpisodeObjs, { transaction, updateOnDuplicate: ['title', 'author', 'description', 'siteURL', 'enclosureURL', 'enclosureType', 'enclosureSize', 'pubDate', 'season', 'episode', 'episodeType', 'duration', 'filePath', 'explicit'] })
}

/**
Expand All @@ -200,14 +216,21 @@ class FeedEpisode extends Model {
}).libraryItem.createdAt

const feedEpisodeObjs = []
let numExisting = 0
for (const book of books) {
const useChapterTitles = this.checkUseChapterTitlesForEpisodes(book)
for (const track of book.trackList) {
feedEpisodeObjs.push(this.getFeedEpisodeObjFromAudiobookTrack(book, earliestLibraryItemCreatedAt, feed, slug, track, useChapterTitles))
// Check for existing episode by filepath
const existingEpisode = feed.feedEpisodes?.find((episode) => {
return episode.filePath === track.metadata.path
})
numExisting = existingEpisode ? numExisting + 1 : numExisting

feedEpisodeObjs.push(this.getFeedEpisodeObjFromAudiobookTrack(book, earliestLibraryItemCreatedAt, feed, slug, track, useChapterTitles, existingEpisode?.id))
}
}
Logger.info(`[FeedEpisode] Creating ${feedEpisodeObjs.length} episodes for feed ${feed.id}`)
return this.bulkCreate(feedEpisodeObjs, { transaction })
Logger.info(`[FeedEpisode] Upserting ${feedEpisodeObjs.length} episodes for feed ${feed.id} (${numExisting} existing)`)
return this.bulkCreate(feedEpisodeObjs, { transaction, updateOnDuplicate: ['title', 'author', 'description', 'siteURL', 'enclosureURL', 'enclosureType', 'enclosureSize', 'pubDate', 'season', 'episode', 'episodeType', 'duration', 'filePath', 'explicit'] })
}

/**
Expand Down

0 comments on commit c8a0592

Please sign in to comment.