diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 00000000..3734ef64 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,4 @@ +{ + "editor.defaultFormatter": "esbenp.prettier-vscode", + "editor.formatOnSave": true +} \ No newline at end of file diff --git a/src/schema-types.ts b/src/schema-types.ts index e8216ca1..9a4fbfe6 100644 --- a/src/schema-types.ts +++ b/src/schema-types.ts @@ -103,6 +103,7 @@ export const enum SyncService { Inoreader, Miniflux, Nextcloud, + NewsBlur, } export interface ServiceConfigs { diff --git a/src/scripts/models/service.ts b/src/scripts/models/service.ts index 99ef1539..4d2fb723 100644 --- a/src/scripts/models/service.ts +++ b/src/scripts/models/service.ts @@ -20,6 +20,7 @@ import { feedbinServiceHooks } from "./services/feedbin"; import { gReaderServiceHooks } from "./services/greader"; import { minifluxServiceHooks } from "./services/miniflux"; import { nextcloudServiceHooks } from "./services/nextcloud"; +import { newsblurServiceHooks } from "./services/newsblur"; export interface ServiceHooks { authenticate?: (configs: ServiceConfigs) => Promise; @@ -51,6 +52,8 @@ export function getServiceHooksFromType(type: SyncService): ServiceHooks { return minifluxServiceHooks; case SyncService.Nextcloud: return nextcloudServiceHooks; + case SyncService.NewsBlur: + return newsblurServiceHooks; default: return {}; } @@ -318,9 +321,7 @@ interface SyncLocalItemsAction { } export type ServiceActionTypes = - | SaveServiceConfigsAction - | SyncWithServiceAction - | SyncLocalItemsAction; + SaveServiceConfigsAction | SyncWithServiceAction | SyncLocalItemsAction; export function saveServiceConfigs(configs: ServiceConfigs): AppThunk { return (dispatch) => { diff --git a/src/scripts/models/services/newsblur.ts b/src/scripts/models/services/newsblur.ts new file mode 100644 index 00000000..aee55341 --- /dev/null +++ b/src/scripts/models/services/newsblur.ts @@ -0,0 +1,505 @@ +/* +This file connects the Newsblur service +to the Fluentflame's API by exporting +the constant newsblurServiceHooks. + +Each hook must make one or several http requests +to the server by using the functions available +in the NewsblurAPI namespace, which represent +abstractions of each Newsblur endpoint used. + +Apart from newsblurServiceHooks and NewsBlurConfigs, +other exported variables and types are exported. +These are meant to be used in the tests: +src/scripts/models/services/test_newsblur.ts + +Newsblur Dictionary: + +| Newsblur | Fluentflame | +|----------|-------------| +| feed | source | +| story | item | +| folders | - | + +*/ + +import { ServiceConfigs, SyncService } from "../../../schema-types"; +import { RootState } from "../../reducer"; +import { generateThumbnailAttrList } from "../../thumb-utils"; +import { htmlDecode } from "../../utils"; +import { RSSItem } from "../item"; +import { SourceRule } from "../rule"; +import { ServiceHooks } from "../service"; +import { RSSSource } from "../source"; +import { ParamsObject, pathParams, toSearchParams } from "./service-utils"; + +export interface NewsBlurConfigs extends ServiceConfigs { + type: SyncService.NewsBlur; + endpoint: URL; // url + username: string; + password: string; +} + +export namespace NewsblurAPI { + // Basic fetch functions (not exported) + + async function fetchGetAPI( + configs: NewsBlurConfigs, + path: string, + params: Record, + ): Promise { + // set url + const url = new URL(configs.endpoint); + url.pathname = path; + // set params + const searchParams = new URLSearchParams(params); + url.search = searchParams.toString(); + // set headers + const headers = new Headers(); + // options + const options: RequestInit = { headers, credentials: "include" }; + // send + const response = await fetch(url, options); + // return or throw + const json: NewsblurResponse = await response.json(); + return json; + } + + async function fetchPostAPI( + configs: NewsBlurConfigs, + path: string, + params: ParamsObject, + ): Promise { + // set url + const url = new URL(configs.endpoint); + url.pathname = path; + // set params + const body = toSearchParams(params); + // set headers + const headers = new Headers(); + headers.set("Content-Type", "application/x-www-form-urlencoded"); + // options + const options: RequestInit = { + method: "POST", + headers: headers, + body: body, + credentials: "include", + }; + // send + const response = await fetch(url, options); + // return or throw + const json: NewsblurPostResponse = await response.json(); + return json; + } + + // Direct API endpoints (exported) + + export async function authenticate( + configs: NewsBlurConfigs, + ): Promise { + const response = await fetchPostAPI(configs, "/api/login", { + username: configs.username, + password: configs.password, + }); + return Boolean(response.authenticated); + } + + export async function fetchFeeds( + configs: NewsBlurConfigs, + ): Promise { + const response = (await fetchGetAPI(configs, "/reader/feeds", { + flat: "true", + })) as NewsblurFeedsResponse; + return Object.values(response.feeds); + } + + export async function fetchStoriesInFeed( + configs: NewsBlurConfigs, + feedId: string | number, + ): Promise { + const response = (await fetchGetAPI( + configs, + pathParams("/reader/feed/:id", { + id: feedId.toString(), + }), + {}, + )) as NewsblurStoriesResponse; + return Object.values(response.stories); + } + + export async function fetchUnreadStoriesInFeed( + configs: NewsBlurConfigs, + feedId: string | number, + ): Promise { + const response = (await fetchGetAPI( + configs, + pathParams("/reader/feed/:id", { + id: feedId.toString(), + }), + { read_filter: "unread" }, + )) as NewsblurStoriesResponse; + return Object.values(response.stories); + } + + export async function fetchAllStarredStories( + configs: NewsBlurConfigs, + ): Promise { + const response = (await fetchGetAPI( + configs, + "/reader/starred_stories", + {}, + )) as NewsblurStoriesResponse; + return Object.values(response.stories); + } + + export async function fetchAllStories( + configs: NewsBlurConfigs, + ): Promise { + const feeds = await NewsblurAPI.fetchFeeds(configs); + const promises = feeds.map((feed) => + NewsblurAPI.fetchStoriesInFeed(configs, feed.id), + ); + return (await Promise.all(promises)).flat(); + } + + export async function setRead( + configs: NewsBlurConfigs, + serviceRef: string, + isRead: boolean, + ) { + if (isRead) { + // For some reason, `.code` is always 1. + // Even when hash is incorrect + await fetchPostAPI(configs, "/reader/mark_story_hashes_as_read", { + story_hash: serviceRef, + }); + } else { + const res = (await fetchPostAPI( + configs, + "/reader/mark_story_hash_as_unread", + { + story_hash: [serviceRef], + }, + )) as NewsblurUnreadResponse; + if (res.code == -1) { + throw new NewsblurError([res.message!]); + } + } + } + + export async function setStar( + configs: NewsBlurConfigs, + serviceRef: string, + isStarred: boolean, + ): Promise { + if (isStarred) { + const res = (await fetchPostAPI( + configs, + "/reader/mark_story_hash_as_starred", + { + story_hash: serviceRef, + }, + )) as NewsblurStarResponse; + if (res.code == -1) { + throw new NewsblurError([res.message]); + } + } else { + const res = (await fetchPostAPI( + configs, + "/reader/mark_story_hash_as_unstarred", + { + story_hash: serviceRef, + }, + )) as NewsblurUnstarResponse; + if (res.code == -1) { + throw new NewsblurError(res.messages); + } + } + } + + export async function markAllAsRead( + configs: NewsBlurConfigs, + ): Promise { + await fetchPostAPI(configs, "/reader/mark_all_as_read", {}); + } +} + +// Types + +export class NewsblurError extends Error { + constructor(errors: string[], options?: ErrorOptions) { + super(errors[0], options); + this.newsblurErrors = errors; + } + newsblurErrors: string[]; +} + +export interface NewsblurResponse { + authenticated: boolean; + user_id: number; +} + +export interface NewsblurPostResponse extends NewsblurResponse { + code: -1 /*error*/ | 1 /*ok*/; +} + +export interface NewsblurAuthResponse extends NewsblurPostResponse { + errors: + | Record + | null /*ok*/ + | { __all__: string[] }; +} + +export interface NewsblurFeedsResponse extends NewsblurResponse { + feeds: Record; +} + +export interface NewsblurStoriesResponse extends NewsblurResponse { + stories: NewsblurStory[]; +} + +export interface NewsblurStarResponse extends NewsblurPostResponse { + /** is empty string `""` if `.code` is `1` */ + message: string; +} + +export interface NewsblurReadResponse extends NewsblurPostResponse { + /** Alwasy 1. Never fails for some reason. */ + code: 1; + story_hashes: string[]; + feed_ids: string[]; +} + +export interface NewsblurUnreadResponse extends NewsblurPostResponse { + /** is empty string `""` if `.code` is `1` */ + code: 1 | -1; + story_hash: string; + feed_id: string; + message: string | undefined; +} + +export interface NewsblurUnstarResponse extends NewsblurPostResponse { + /** is empty array `[]` if `.code` is `1` */ + messages: string[]; +} + +/** A string with a date in format YYYY-MM-DDThh:mm:ss (T is just a T) */ +type dateString = string; + +interface NewsblurFeed { + id: number; + feed_title: string; + feed_address: string; + feed_link: string; + last_story_date: dateString | null; +} + +interface NewsblurStory { + story_hash: string; + story_timestamp: string; + story_authors: string; + score: number; + read_status: 0 | 1; + id: string; + story_feed_id: number; // serviceRef of RSSSource + story_title: string; + story_content: string; + starred: boolean; +} + +// Hooks (the api) +export const newsblurServiceHooks: ServiceHooks = { + authenticate: async (serviceConfigs) => { + const configs = serviceConfigs as NewsBlurConfigs; + try { + return await NewsblurAPI.authenticate(configs); + } catch (e) { + console.error(e); + return false; + } + }, + + /////////////////// + // GET REQUESTS // + /////////////////// + + /** get remote sources */ + updateSources: () => async (_, getState: () => RootState) => { + const configs = getState().service as NewsBlurConfigs; + + // fetch + const sources: RSSSource[] = await NewsblurAPI.fetchFeeds(configs).then( + (feeds) => + feeds.map((f) => { + const source = new RSSSource(f.feed_address, f.feed_title); + source.serviceRef = String(f.id); + return source; + }), + ); + + return [sources, new Map() /* No groups in Newsblur */]; + }, + + /** get and set remote unreads and starreds */ + syncItems: () => async (_, getState) => { + const configs = getState().service as NewsBlurConfigs; + const unread = new Set(); + const starred = new Set(); + + // get all rss sources with unread posts. Call only once a minute !!! + const unreadsPromise: Promise = (async () => { + const feeds = await NewsblurAPI.fetchFeeds(configs); + + // get unread + const unreadPromises: Promise[] = // keep + Object.values(feeds).map((feed) => + // call to each feed + NewsblurAPI.fetchUnreadStoriesInFeed(configs, feed.id).then( + (stories) => stories.map((story) => story.id), + ), + ); + return (await Promise.all(unreadPromises)).flat(); + })(); + + // get starred + let starredsPromise = NewsblurAPI.fetchAllStarredStories(configs).then( + (stories) => stories.map((story) => story.id), + ); + + // wait for values + for (const unreadId of await unreadsPromise) { + unread.add(unreadId); + } + for (const id of await starredsPromise) { + starred.add(id); + } + + return [unread, starred]; + }, + + // get and set remote items + fetchItems: () => async (_, getState) => { + const state = getState(); + const configs = state.service as NewsBlurConfigs; + + // get sources that possess ref/id given by service, associate new items + const sourceMap = new Map( + Object.values(state.sources) + .filter((s) => s.serviceRef) + .map((s) => [s.serviceRef, s]), + ); + + // parse stories + const stories = await NewsblurAPI.fetchAllStories(configs).then( + (stories) => + stories.map((story): RSSItem => { + const source = sourceMap.get(String(story.story_feed_id)); + + // parse item + let parsedItem = { + source: source?.sid, + title: story.story_title, + link: story.id, + date: new Date(parseInt(story.story_timestamp)), + fetchedDate: new Date(), + content: story.story_content, + snippet: htmlDecode(story.story_content).trim(), + creator: story.story_authors, + hasRead: Boolean(story.read_status == 1), + starred: Boolean(story.starred), + hidden: false, + notify: false, + serviceRef: String(story.story_hash), + // will allow page to load, and get the images + // async eventually + thumbnailJobs: generateThumbnailAttrList({ + targetLink: story.id, // url + content: story.story_content, + }), + } as RSSItem; + + // Apply rules and sync back + // prettier-ignore + if (source?.rules) { + SourceRule.applyAll(source.rules, parsedItem); + const readChanged = Boolean(story.read_status == 1) !== parsedItem.hasRead; + const starChanged = Boolean(story.starred) !== parsedItem.starred; + if (readChanged) NewsblurAPI.setRead(configs, parsedItem.serviceRef, parsedItem.hasRead); + if (starChanged) NewsblurAPI.setStar(configs, parsedItem.serviceRef, parsedItem.starred); + } + + return parsedItem; + }), + ); + + // collect + let parsedItems: RSSItem[] = (await Promise.all(stories)).flat(); + + return [/*RSSItem[]*/ parsedItems, /*ServiceConfigs*/ configs]; + }, + + /////////////////// + // POST REQUESTS // + /////////////////// + + markAllRead: (_sids, date, _before) => async (_, getState) => { + const state = getState(); + const configs = state.service as NewsBlurConfigs; + + if (date == null) { + // mark all + await NewsblurAPI.markAllAsRead(configs); + } else { + // mark only those after date + const requests = state.feeds[state.page.feedId].iids + .map((iid) => state.items[iid]) + .filter((i) => !i.hasRead && i.date.getTime() >= date.getTime()) + .map(async (item) => { + // prettier-ignore + if (item.serviceRef) { + await NewsblurAPI.setRead(configs, item.serviceRef, true); + } + }); + await Promise.all(requests); + } + }, + + markRead: (item: RSSItem) => async (_, getState) => { + // Note: could be optimized if instead of making + // one request for each RSSItem, it makes one request + // with the hashes of all RSSItem's at once + const state = getState(); + const configs = state.service as NewsBlurConfigs; + + if (item.serviceRef) { + await NewsblurAPI.setRead(configs, item.serviceRef, true); + } + }, + + markUnread: (item: RSSItem) => async (_, getState) => { + const state = getState(); + const configs = state.service as NewsBlurConfigs; + + if (item.serviceRef) { + await NewsblurAPI.setRead(configs, item.serviceRef, false); + } + }, + + star: (item: RSSItem) => async (_, getState) => { + const state = getState(); + const configs = state.service as NewsBlurConfigs; + + if (item.serviceRef) { + await NewsblurAPI.setStar(configs, item.serviceRef, true); + } + }, + + unstar: (item: RSSItem) => async (_, getState) => { + const state = getState(); + const configs = state.service as NewsBlurConfigs; + + if (item.serviceRef) { + await NewsblurAPI.setStar(configs, item.serviceRef, false); + } + }, +}; diff --git a/src/scripts/models/services/service-utils.ts b/src/scripts/models/services/service-utils.ts index ff2d9a26..59323269 100644 --- a/src/scripts/models/services/service-utils.ts +++ b/src/scripts/models/services/service-utils.ts @@ -24,3 +24,46 @@ export function getItemEntries( }) .toArray(); } + +/** + * An easier way to creating URLSearchParams() when + * one has to pass arrays in this format + * + * Instead of writing "array=1&array=2&array=3", one + * can pass an object: + * ```ts + * { array: ["1", "2", "3"] } + * ``` + * and it will be converted to the same url encoded form + * + */ +export function toSearchParams(object: ParamsObject): URLSearchParams { + const params = new URLSearchParams(); + for (const key in object) { + const value = object[key]; + if (Array.isArray(value)) { + for (const innerValue of value) { + params.append(key, innerValue.toString()); + } + } else { + params.set(key, value.toString()); + } + } + return params; +} +export type ParamsObject = Record; + +/** + * A helper function to add path-params to a path + * @param path - for example `/profile/:id` + * @param params - for example `{ id: "username" }` + * @returns + */ +export function pathParams(path: string, params: Record) { + let finalPath = path; + for (const param in params) { + const value = params[param]; + finalPath = finalPath.replace(`:${param}`, encodeURIComponent(value)); + } + return finalPath; +} diff --git a/tests/scripts/models/services/test_newsblur.ts b/tests/scripts/models/services/test_newsblur.ts new file mode 100644 index 00000000..82ab00ba --- /dev/null +++ b/tests/scripts/models/services/test_newsblur.ts @@ -0,0 +1,146 @@ +import { assert } from "chai"; +import { SyncService } from "../../../../src/schema-types"; +import { + type NewsblurAuthResponse, + type NewsBlurConfigs, + NewsblurFeedsResponse, + newsblurServiceHooks, +} from "../../../../src/scripts/models/services/newsblur"; +import { afterEach } from "mocha"; +import { RSSSource } from "../../../../src/scripts/models/source"; + +// Configs + +const CONFIGS: NewsBlurConfigs = { + username: "yeah", + password: "maracuyeah", + endpoint: new URL("https://newsblur.com/"), + type: SyncService.NewsBlur, +}; + +// Possible responses + +const NEWSBLUR_AUTH_SUCCESS: NewsblurAuthResponse = { + authenticated: true, + code: 1, + errors: null, + user_id: 10, +}; + +const NEWSBLUR_AUTH_ERROR: NewsblurAuthResponse = { + authenticated: false, + code: -1, + errors: { + __all__: ["Whoopsy-daisy, wrong password. Try again."], + }, + user_id: 10, +}; + +const NEWSBLUR_UPDATE_SOURCES: NewsblurFeedsResponse = { + feeds: { + "8268588": { + id: 8268588, + feed_title: "xkcd.com", + feed_address: "https://xkcd.com/atom.xml", + feed_link: "https://xkcd.com/", + last_story_date: "2026-07-06T00:00:00", + }, + "8356326": { + id: 8356326, + feed_title: "Pivot", + feed_address: "https://pivot.quebec/feed/", + feed_link: "https://pivot.quebec/", + last_story_date: "2026-07-06T20:40:59", + }, + }, + user_id: 753313, + authenticated: true, +}; + +// Mock + +let original = { + fetch: null as null | typeof fetch, + consoleError: null as null | typeof console.error, +}; +const mockFetch = (fn: () => Promise) => { + if (original.fetch == null) { + original.fetch = fetch; + global.fetch = fn; + } else { + throw new Error("fetch already mocked"); + } +}; +const mockConsoleError = (fn: typeof console.error) => { + if (original.consoleError == null) { + original.consoleError = console.error; + console.error = fn; + } else { + throw new Error("console.error already mocked"); + } +}; +const unmock = () => { + if (original.fetch != null) { + global.fetch = original.fetch; + original.fetch = null; + } + if (original.consoleError != null) { + console.error = original.consoleError; + original.consoleError = null; + } +}; + +// Tests + +describe("newsblurServiceHooks", () => { + afterEach(unmock); + // 1. can authenticate + it("can authenticate", async () => { + mockFetch(() => + Promise.resolve( + new Response(JSON.stringify(NEWSBLUR_AUTH_SUCCESS)), + ), + ); + const authenticated = + await newsblurServiceHooks.authenticate?.(CONFIGS); + assert.equal(authenticated, true); + }); + it("wont authenticate", async () => { + mockFetch(() => + Promise.reject(new Response(JSON.stringify(NEWSBLUR_AUTH_ERROR))), + ); + mockConsoleError((err) => { + assert.exists(err); + }); + const authenticated = + await newsblurServiceHooks.authenticate?.(CONFIGS); + assert.equal(authenticated, false); + }); + // 2. can get sources + it("can update sources", async () => { + mockFetch(() => + Promise.resolve( + new Response(JSON.stringify(NEWSBLUR_UPDATE_SOURCES)), + ), + ); + const updater = newsblurServiceHooks.updateSources?.(); + assert.exists(updater); + const mockDispatch: any = (_d: any, _payload: any) => { + return null; + }; + const mockGetState: () => any = () => { + return { + service: CONFIGS, + }; + }; + const result = await updater(mockDispatch, mockGetState, null); + assert.equal(result.length, 2); + const [sources] = result; + assert.equal(sources.length, 2); + assert.equal(sources[0].name, "xkcd.com"); + assert.equal(sources[1].serviceRef, "8356326"); + }); + // 3. can sync with service + // 4. can fetch from/to service + // 5. can mark unreads and start +}); diff --git a/tsconfig.json b/tsconfig.json index c35c3f0f..b4bffb5a 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -5,6 +5,8 @@ "esModuleInterop": true, "target": "ESNext", "module": "ESNext", - "moduleResolution": "Bundler" - } + "moduleResolution": "Bundler", + "types": ["mocha", "node"] + }, + "include": ["./tests/**/*"] }