Files
Tom Moor 6231f01860 fix: Forced fetches dropped or stale while another fetch is in flight (#14573)
* fix: Run forced document tree fetches that arrive while a fetch is in flight

* fix: Run forced store fetches that arrive while a fetch is in flight

* test
2026-11-12 23:07:35 -05:00

694 lines
19 KiB
TypeScript

import commandScore from "command-score";
import invariant from "invariant";
import {
deburr,
filter,
find,
flatten,
lowerFirst,
orderBy,
} from "es-toolkit/compat";
import {
action,
computed,
makeObservable,
observable,
runInAction,
} from "mobx";
import pluralize from "pluralize";
import { isUUID } from "validator";
import { Pagination } from "@shared/constants";
import { type JSONObject } from "@shared/types";
import type RootStore from "~/stores/RootStore";
import type Policy from "~/models/Policy";
import type ArchivableModel from "~/models/base/ArchivableModel";
import { KeyedCoalescedFetch } from "~/models/base/CoalescedFetch";
import type Model from "~/models/base/Model";
import { LifecycleManager } from "~/models/decorators/Lifecycle";
import { getInverseRelationsForModelClass } from "~/models/decorators/Relation";
import type { Searchable } from "~/models/interfaces/Searchable";
import type { PaginationParams, PartialExcept, Properties } from "~/types";
import { client } from "~/utils/ApiClient";
import { AuthorizationError, NotFoundError } from "~/utils/errors";
import ParanoidModel from "~/models/base/ParanoidModel";
import StorePersistence from "./StorePersistence";
// A concrete model constructor along with the static members of Model.
type ModelClass<T extends Model> = (new (
fields: PartialExcept<T, "id">,
store: Store<T>
) => T) &
Pick<typeof Model, keyof typeof Model>;
type ListPredicate<T> =
| ((value: T, index: number, collection: ArrayLike<T>) => boolean)
| PropertyKey
| [PropertyKey, unknown]
| { [P in keyof T]?: T[P] extends object ? object : T[P] };
export enum RPCAction {
Info = "info",
List = "list",
Create = "create",
Update = "update",
Delete = "delete",
Count = "count",
}
export const PAGINATION_SYMBOL = Symbol.for("pagination");
export type PaginatedResponse<T> = T[] & {
[PAGINATION_SYMBOL]?: {
total: number;
limit: number;
offset: number;
nextPath: string;
};
};
// oxlint-disable-next-line no-explicit-any
export type FetchPageParams = PaginationParams & Record<string, any>;
/**
* A store that related models found in an API response can be added to.
*/
interface RelatedStore {
responseKey: string;
add(item: PartialExcept<Model, "id">): unknown;
}
export default abstract class Store<T extends Model> {
@observable
data: Map<string, T> = new Map();
@observable
isFetching = false;
@observable
isSaving = false;
@observable
isLoaded = false;
/** Calls to `Model.loadRelations` that are in flight, keyed by model id. */
relationFetches = new KeyedCoalescedFetch<unknown>();
model: ModelClass<T>;
modelName: string;
apiEndpoint: string;
/**
* The key under which this store's models appear in API responses that
* return more than one kind of model, defaults to the API endpoint name.
*/
responseKey: string;
/**
* Whether the store's data should be persisted to IndexedDB, and restored
* at boot, once persistence is enabled for the authenticated team.
*/
persistable = false;
rootStore: RootStore;
protected persistence?: StorePersistence<T>;
actions = [
RPCAction.Info,
RPCAction.List,
RPCAction.Create,
RPCAction.Update,
RPCAction.Delete,
];
constructor(rootStore: RootStore, model: ModelClass<T>) {
this.rootStore = rootStore;
this.model = model;
this.modelName = model.modelName;
if (!this.apiEndpoint) {
this.apiEndpoint = pluralize(lowerFirst(model.modelName));
}
if (!this.responseKey) {
this.responseKey = this.apiEndpoint;
}
makeObservable(this);
}
@action
clear() {
this.data.clear();
this.fetchAllResults.clear();
void this.persistence?.clear();
}
/**
* Enables persistence of this store's data to IndexedDB, scoped to the
* given team, and hydrates any previously persisted records into the store.
* A no-op if the store is not persistable, persistence is unsupported, or it
* is already enabled, so it is safe to call on every store and more than once.
*
* @param teamId the ID of the team the persisted data belongs to.
* @returns a promise that resolves when hydration is complete.
*/
async enablePersistence(teamId: string): Promise<void> {
if (
!this.persistable ||
this.persistence ||
!StorePersistence.isSupported
) {
return;
}
this.persistence = new StorePersistence<T>(this, teamId);
await this.persistence.hydrate();
}
/**
* Disables persistence of this store's data, closing the connection to the
* database without removing anything that was already persisted. Safe to call
* whether or not persistence is currently enabled.
*/
disablePersistence(): void {
this.persistence?.disable();
this.persistence = undefined;
}
addPolicies = (policies: Policy[]) => {
policies?.forEach((policy) => this.rootStore.policies.add(policy));
};
/**
* Finds the items that match the given query, ordered by relevance.
*
* @param query the search query, all items are returned when it is empty.
* @param options the maximum number of results, and a weight applied to the
* relevance score of each item, for example how familiar it is to the user.
* @returns the matching items.
*/
findByQuery = (
query: string,
options?: { maxResults?: number; weight?: (item: T) => number }
): T[] => {
const normalized = deburr((query ?? "").trim().toLocaleLowerCase());
const weight = options?.weight;
if (!normalized) {
const items = this.orderedData.filter(
(item: T & Searchable) => !item.searchSuppressed
);
return (
weight ? items.sort((a, b) => weight(b) - weight(a)) : items
).slice(0, options?.maxResults);
}
return this.orderedData
.filter((item: T & Searchable) => {
if (item.searchSuppressed) {
return false;
}
if ("searchContent" in item) {
const seachables =
typeof item.searchContent === "string"
? [item.searchContent]
: item.searchContent;
return seachables.some((searchable) =>
deburr(searchable.toLocaleLowerCase()).includes(normalized)
);
}
throw new Error("Item does not implement Searchable interface");
})
.map((item: T & Searchable) => {
const seachables =
typeof item.searchContent === "string"
? [item.searchContent]
: item.searchContent;
return {
score:
(seachables
.map((searchable) => commandScore(normalized, searchable))
.reduce((a, b) => a + b, 0) /
seachables.length) *
(weight?.(item) ?? 1),
item,
};
})
.sort((a, b) => b.score - a.score)
.map(({ item }) => item)
.slice(0, options?.maxResults);
};
@action
add = (item: PartialExcept<T, "id"> | T): T => {
const ModelClass = this.model;
if (!(item instanceof ModelClass)) {
const existingModel = this.data.get(item.id);
if (existingModel) {
existingModel.updateData(item);
this.persistence?.persist(existingModel.id);
return existingModel;
}
const newModel: T = new ModelClass(item, this);
this.data.set(newModel.id, newModel);
this.persistence?.persist(newModel.id);
this.fetchAllResults.clear();
return newModel;
}
if (!this.data.has(item.id)) {
this.fetchAllResults.clear();
}
this.data.set(item.id, item);
this.persistence?.persist(item.id);
return item;
};
/**
* Remove a model, and any models that cascade from it, from the store.
*
* @param id the ID of the model to remove.
* @param options.permanent whether soft-deletable models should be evicted
* from the store entirely rather than marked as deleted. Use when the model
* is gone for good, or is no longer accessible to the current user.
*/
@action
remove(id: string, options?: { permanent?: boolean }): void {
const model = this.data.get(id);
if (!model) {
return;
}
const inverseRelations = getInverseRelationsForModelClass(this.model);
inverseRelations.forEach((relation) => {
const store = this.rootStore.getStoreForModelName(relation.modelName);
if ("orderedData" in store) {
const items = (store.orderedData as Model[]).filter(
(item) => item[relation.idKey] === id
);
items.forEach((item) => {
let deleteBehavior = relation.options.onDelete;
if (typeof relation.options.onDelete === "function") {
deleteBehavior = relation.options.onDelete(item);
}
if (deleteBehavior === "cascade") {
store.remove(item.id, options);
} else if (deleteBehavior === "null") {
Reflect.set(item, relation.idKey, null);
}
});
}
});
// Remove associated policies automatically, not defined through Relation decorator.
if (this.modelName !== "Policy") {
this.rootStore.policies.remove(id);
}
LifecycleManager.executeHooks(model.constructor, "beforeRemove", model);
if (model instanceof ParanoidModel && !options?.permanent) {
model.deletedAt = new Date().toISOString();
} else {
this.data.delete(id);
}
LifecycleManager.executeHooks(model.constructor, "afterRemove", model);
this.fetchAllResults.clear();
// Persists the soft-deleted model, or removes the persisted record if the
// model was hard-deleted from the store.
this.persistence?.persist(id);
}
@action
addToArchive(item: ArchivableModel): void {
const inverseRelations = getInverseRelationsForModelClass(this.model);
inverseRelations.forEach((relation) => {
const store = this.rootStore.getStoreForModelName(relation.modelName);
if ("orderedData" in store) {
const items = (store.orderedData as ArchivableModel[]).filter(
(data) => data[relation.idKey] === item.id
);
items.forEach((item) => {
let archiveBehavior = relation.options.onArchive;
if (typeof relation.options.onArchive === "function") {
archiveBehavior = relation.options.onArchive(item);
}
if (archiveBehavior === "cascade") {
store.addToArchive(item);
} else if (archiveBehavior === "null") {
Reflect.set(item, relation.idKey, null);
}
});
}
});
// Remove associated policies automatically, not defined through Relation decorator.
if (this.modelName !== "Policy") {
this.rootStore.policies.remove(item.id);
}
item.archivedAt = new Date().toISOString();
(this as unknown as Store<ArchivableModel>).add(item);
}
/**
* Remove all items in the store that match the predicate.
*
* @param predicate A function that returns true if the item matches, or an object with the properties to match.
*/
removeAll = (predicate: Parameters<typeof this.filter>[0]) => {
this.filter(predicate).forEach((item) => this.remove(item.id));
};
save(params: Properties<T>, options: JSONObject = {}): Promise<T> {
const { isNew, ...rest } = options;
if (isNew || !("id" in params) || !params.id) {
return this.create(params, rest);
}
return this.update(params, rest);
}
/**
* Get a single item from the store that matches the ID.
*
* @param id The ID of the item to get.
*/
get(id: string): T | undefined {
return id ? this.data.get(id) : undefined;
}
@action
async create(params: Properties<T>, options?: JSONObject): Promise<T> {
if (!this.actions.includes(RPCAction.Create)) {
throw new Error(`Cannot create ${this.modelName}`);
}
this.isSaving = true;
try {
const res = await client.post(`/${this.apiEndpoint}.create`, {
...params,
...options,
});
return runInAction(() => {
invariant(res?.data, "Data should be available");
this.addPolicies(res.policies);
return this.add(res.data);
});
} finally {
this.isSaving = false;
}
}
@action
async update(params: Properties<T>, options?: JSONObject): Promise<T> {
if (!this.actions.includes(RPCAction.Update)) {
throw new Error(`Cannot update ${this.modelName}`);
}
this.isSaving = true;
try {
const res = await client.post(`/${this.apiEndpoint}.update`, {
...params,
...options,
});
return runInAction(() => {
invariant(res?.data, "Data should be available");
this.addPolicies(res.policies);
return this.add(res.data);
});
} finally {
this.isSaving = false;
}
}
@action
async delete(item: T, options: JSONObject = {}) {
if (!this.actions.includes(RPCAction.Delete)) {
throw new Error(`Cannot delete ${this.modelName}`);
}
if (item.isNew) {
return this.remove(item.id);
}
this.isSaving = true;
try {
await client.post(`/${this.apiEndpoint}.delete`, {
id: item.id,
...options,
});
return this.remove(item.id);
} finally {
this.isSaving = false;
}
}
/**
* Whether individual models can be fetched by ID from this store.
*/
get canFetchById(): boolean {
return this.actions.includes(RPCAction.Info);
}
@action
async fetch(
id: string,
options: JSONObject = {},
accessor = (res: unknown) => (res as { data: PartialExcept<T, "id"> }).data
): Promise<T> {
if (!this.actions.includes(RPCAction.Info)) {
throw new Error(`Cannot fetch ${this.modelName}`);
}
const item = this.get(id);
if (item && !options.force) {
return item;
}
return this.fetches.run(
id,
async () => {
runInAction(() => {
this.isFetching = true;
});
try {
const res = await client.post(`/${this.apiEndpoint}.info`, {
id,
});
return runInAction(() => {
invariant(res?.data, "Data should be available");
this.addPolicies(res.policies);
return this.add(accessor(res));
});
} catch (err) {
if (
err instanceof AuthorizationError ||
err instanceof NotFoundError
) {
this.remove(id);
}
throw err;
} finally {
runInAction(() => {
this.isFetching = false;
});
}
},
Boolean(options.force)
);
}
// Not annotated: subclasses replace this field, and MobX makes an annotated
// field non-writable. The state changes run in runInAction already.
fetchPage = async (
params?: FetchPageParams
): Promise<PaginatedResponse<T>> => {
if (!this.actions.includes(RPCAction.List)) {
throw new Error(`Cannot list ${this.modelName}`);
}
return this.fetchPaginated(`/${this.apiEndpoint}.list`, params);
};
/**
* Fetches every page of the list endpoint.
*
* @param params the request parameters.
* @returns all fetched items.
*/
@action
fetchAll = async (
params: FetchPageParams = {}
): Promise<PaginatedResponse<T>> => {
const limit = params.limit ?? Pagination.defaultLimit;
const response = await this.fetchPage({ ...params, limit });
invariant(
response[PAGINATION_SYMBOL],
"Pagination information not available in response"
);
const pages = Math.ceil(response[PAGINATION_SYMBOL].total / limit);
const fetchPages = [];
for (let page = 1; page < pages; page++) {
fetchPages.push(
this.fetchPage({ ...params, offset: page * limit, limit })
);
}
const results = flatten([
response,
...(fetchPages.length ? await Promise.all(fetchPages) : []),
]);
if (params.withRelations) {
await Promise.all(
this.orderedData.map((integration) => integration.loadRelations())
);
}
this.fetchAllResults.set(JSON.stringify(params), results);
return results;
};
/**
* Fetches every page of the list endpoint unless the same params were
* fetched before and no model has been added to or removed from the store
* since. Use for reference lists that are loaded on mount.
*
* @param params the request parameters.
* @returns all fetched items.
*/
fetchAllIfNeeded = async (
params: FetchPageParams = {}
): Promise<PaginatedResponse<T>> =>
this.fetchAllResults.get(JSON.stringify(params)) ??
(await this.fetchAll(params));
@computed
get orderedData(): T[] {
return orderBy(Array.from(this.data.values()), "createdAt", "desc");
}
/**
* Find an item in the store matching the given predicate.
*
* @param predicate A function that returns true if the item matches, or an object with the properties to match.
*/
find = (predicate: ListPredicate<T>): T | undefined =>
find(this.orderedData, predicate);
/**
* Filter items in the store matching the given predicate.
*
* @param predicate A function that returns true if the item matches, or an object with the properties to match.
*/
filter = (predicate: ListPredicate<T>): T[] =>
filter(this.orderedData, predicate);
/**
* Get an item from the store by ID, or by the urlId at the end of a URL
* slug. A UUID cannot end with a urlId, so the slower search by urlId is
* skipped for UUIDs, and observers do not track every item in the store.
*
* @param id the ID or URL slug of the item.
* @returns the matching item, if it is in the store.
*/
protected getByIdOrUrlId(id: string): T | undefined {
const item = this.data.get(id);
if (item || !id || isUUID(id)) {
return item;
}
return this.orderedData.find(
(model) =>
"urlId" in model &&
typeof model.urlId === "string" &&
id.endsWith(model.urlId)
);
}
/**
* Fetch a page of items from the API and add them to the store, along with
* any related models in the response that belong to other stores. Items are
* read from the response data directly when it is an array, otherwise from
* the key of each store.
*
* @param endpoint the API endpoint to request.
* @param params the request parameters.
* @param related the stores that related models in the response are added to.
* @returns the fetched items, with pagination information attached.
*/
protected async fetchPaginated(
endpoint: string,
params?: FetchPageParams,
related: RelatedStore[] = []
): Promise<PaginatedResponse<T>> {
runInAction(() => {
this.isFetching = true;
});
try {
const res = await client.post(endpoint, params);
invariant(res?.data, "Data not available");
let response: PaginatedResponse<T> = [];
runInAction(() => {
this.addPolicies(res.policies);
related.forEach((store) => {
res.data[store.responseKey]?.forEach(
(item: PartialExcept<Model, "id">) => store.add(item)
);
});
const items = Array.isArray(res.data)
? res.data
: res.data[this.responseKey];
response = items.map(this.add);
this.isLoaded = true;
});
response[PAGINATION_SYMBOL] = res.pagination;
return response;
} finally {
runInAction(() => {
this.isFetching = false;
});
}
}
/** Requests by `fetch` that are in flight, keyed by model id. */
protected fetches = new KeyedCoalescedFetch<T>();
/** Results of `fetchAll` calls keyed by their params, see `fetchAllIfNeeded`. */
private fetchAllResults = new Map<string, PaginatedResponse<T>>();
}