diff --git a/src/App/marketCap/marketCap.service.ts b/src/App/marketCap/marketCap.service.ts index eef28a6b..fe31bb85 100644 --- a/src/App/marketCap/marketCap.service.ts +++ b/src/App/marketCap/marketCap.service.ts @@ -1,12 +1,13 @@ /* eslint-disable no-continue */ /* eslint-disable no-underscore-dangle */ -import { DataSource } from 'typeorm' +import { DataSource, In } from 'typeorm' import { Inject, Injectable, Logger } from '@nestjs/common' import { CACHE_MANAGER } from '@nestjs/cache-manager' import { Cache } from 'cache-manager' import { ConfigService } from '@nestjs/config' import { BASE_URL_COINGECKO_API, + baseCoingeckoUrl, MARKETCAP_SEARCH_CACHE_KEY, MarketCapInputs, MarketCapSearchInputs, @@ -37,50 +38,11 @@ const noContentWiki = { } as unknown as Partial interface RankPageWiki { - wiki: Wiki + wiki: Wiki | null founders: (Wiki | null)[] blockchain: (Wiki | null)[] } -interface MarketCapMapping { - coingeckoId: string - wikiId: string -} - -interface WikiRawData { - id: string - title: string - ipfs: string - images: { id: string; type: string }[] - metadata: { id: string; value: string }[] - created: Date - linkedWikis: { founders?: string[]; blockchains?: string[] } - tag_ids: ({ id: string } | null)[] -} - -interface LinkedWikiData { - id: string - title: string - ipfs: string - images: { id: string; type: string }[] -} - -interface EventData { - wikiId: string - type: string - date: Date - title?: string - description?: string - link?: string - multiDateStart?: Date - multiDateEnd?: Date -} - -interface ProcessedWikiData extends Omit { - tags: { id: string }[] - events?: EventData[] -} - @Injectable() class MarketCapService { private readonly logger = new Logger(MarketCapService.name) @@ -103,169 +65,377 @@ class MarketCapService { this.RANK_LIMIT = this.configService.get('RANK_LIMIT', 2500) } - private async findWikisBulk( - ids: string[], + private async findWikis( category: string, - ): Promise> { - const resultsMap = new Map() - if (!ids.length) return resultsMap + items: { index: number; id: string; rankPageWiki: RankPageWiki | null }[], + ): Promise< + { index: number; id: string; rankPageWiki: RankPageWiki | null }[] + > { + const missingItems = await this.checkCacheAndGetMissing(items) + + if (missingItems.length === 0) { + return items + } + + const idMapping = await this.getIdMapping(category, missingItems) + const wikiResults = await this.fetchWikis(category, idMapping) + const enrichedResults = await this.enrichWikisWithRelatedData(wikiResults) + + await this.cacheResults(missingItems, enrichedResults) - const cachedResults = await Promise.all( - ids.map((id) => this.cacheManager.get(id)), + return this.mergeResults(items, enrichedResults) + } + + private async checkCacheAndGetMissing( + items: { index: number; id: string; rankPageWiki: RankPageWiki | null }[], + ): Promise<{ index: number; id: string }[]> { + const missingItems: { index: number; id: string }[] = [] + + const cacheLookups = items.map((item) => + this.cacheManager.get(item.id), ) - const uncachedIds = ids.filter((id, i) => { - if (cachedResults[i]) resultsMap.set(id, cachedResults[i]!) - return !cachedResults[i] + const cachedResults = await Promise.all(cacheLookups) + items.forEach((item, i) => { + const cachedWiki = cachedResults[i] + if (cachedWiki) { + item.rankPageWiki = cachedWiki + } else { + missingItems.push({ index: item.index, id: item.id }) + } }) - if (!uncachedIds.length) return resultsMap - const coinType = category === 'cryptocurrencies' ? 'coins' : 'nft' - const profileUrls = uncachedIds.map( - (id) => `https://www.coingecko.com/en/${coinType}/${id}`, - ) + return missingItems + } - const [marketCapMappings, wikisRaw] = await Promise.all([ - this.dataSource.query( - `SELECT "coingeckoId", "wikiId" FROM market_cap_ids WHERE "coingeckoId" = ANY($1::text[]) AND kind = $2`, - [uncachedIds, category], - ) as Promise, - this.dataSource.query( - `SELECT w.id, w.title, w.ipfs, w.images, w.metadata, w.created, w."linkedWikis", - array_agg(DISTINCT jsonb_build_object('id', t.id)) FILTER (WHERE t.id IS NOT NULL) as tag_ids - FROM wiki w - LEFT JOIN wiki_tags_tag wt ON wt."wikiId" = w.id - LEFT JOIN tag t ON t.id = wt."tagId" - WHERE w.hidden = false AND (w.id = ANY($1::text[]) OR EXISTS ( - SELECT 1 FROM json_array_elements(w.metadata) AS meta - WHERE meta->>'id' = 'coingecko_profile' AND meta->>'value' = ANY($2::text[]) - )) - GROUP BY w.id`, - [uncachedIds, profileUrls], - ) as Promise, - ]) + private async getIdMapping( + category: string, + missingItems: { index: number; id: string }[], + ): Promise<{ coingeckoId: string; wikiId: string; index: number }[]> { + const marketCapIdRepository = this.dataSource.getRepository(MarketCapIds) - const cgToWikiMap = new Map( - marketCapMappings.map((m: MarketCapMapping) => [m.coingeckoId, m.wikiId]), - ) - const wikiMap = new Map( - wikisRaw.map((w: WikiRawData) => [ - w.id, - { - ...w, - tags: - w.tag_ids?.filter((tag): tag is { id: string } => tag !== null) || - [], - }, - ]), + const marketCapIds = await marketCapIdRepository.find({ + where: { + coingeckoId: In(missingItems.map((m) => m.id)), + kind: category as RankType, + }, + select: ['wikiId', 'coingeckoId'], + }) + + const marketCapIdMap = new Map( + marketCapIds.map((m) => [m.coingeckoId, m.wikiId]), ) - const urlToIdMap = new Map( - wikisRaw.flatMap((w: WikiRawData) => { - const cg = w.metadata?.find( - (m: { id: string; value: string }) => m.id === 'coingecko_profile', + return missingItems.map((mis) => ({ + coingeckoId: mis.id, + wikiId: marketCapIdMap.get(mis.id) || '', + index: mis.index, + })) + } + + private async fetchWikis( + category: string, + idMapping: { coingeckoId: string; wikiId: string; index: number }[], + ): Promise<{ index: number; wiki: Wiki | null }[]> { + const wikiRepository = this.dataSource.getRepository(Wiki) + const baseQuery = wikiRepository + .createQueryBuilder('wiki') + .select([ + 'wiki.id', + 'wiki.title', + 'wiki.ipfs', + 'wiki.metadata', + 'wiki.images', + ]) + + const wikiQuery = () => + baseQuery + .clone() + .addSelect('wiki.linkedWikis') + .leftJoinAndSelect('wiki.tags', 'tags') + + const matchedWikiIds = idMapping.filter((f) => f.wikiId) + const notMatchedWikiIds = idMapping.filter((f) => !f.wikiId) + + const wikiResults: { index: number; wiki: Wiki | null }[] = [] + + if (matchedWikiIds.length > 0) { + const wikis = await this.fetchWikisByIds( + wikiQuery(), + matchedWikiIds.map((f) => f.wikiId), + ) + wikiResults.push( + ...matchedWikiIds.map((matched) => ({ + index: matched.index, + wiki: wikis.find((w) => w.id === matched.wikiId) || null, + })), + ) + } + + if (notMatchedWikiIds.length > 0) { + const wikis = await this.fetchWikisByIds( + wikiQuery(), + notMatchedWikiIds.map((f) => f.coingeckoId), + ) + const foundIds = new Set(wikis.map((w) => w.id)) + const stillNotFound = notMatchedWikiIds.filter( + (f) => !foundIds.has(f.coingeckoId), + ) + + wikiResults.push( + ...notMatchedWikiIds.map((matched) => ({ + index: matched.index, + wiki: wikis.find((w) => w.id === matched.coingeckoId) || null, + })), + ) + + if (stillNotFound.length > 0) { + await this.fetchWikisByCoingeckoUrl( + category, + stillNotFound, + wikiQuery(), + wikiResults, ) - return cg?.value ? [[cg.value, w.id]] : [] - }), + } + } + + return wikiResults + } + + private async fetchWikisByIds(query: any, ids: string[]): Promise { + return await query + .where('wiki.id IN (:...ids) AND wiki.hidden = false', { ids }) + .getMany() + } + + private async fetchWikisByCoingeckoUrl( + category: string, + notFoundItems: { coingeckoId: string; index: number }[], + wikiQuery: any, + wikiResults: { index: number; wiki: Wiki | null }[], + ): Promise { + const urlsToSearch = notFoundItems.map( + (id) => + `${baseCoingeckoUrl}/${category === 'cryptocurrencies' ? 'coins' : 'nft'}/${id.coingeckoId}`, ) - const getWiki = (id: string): ProcessedWikiData | undefined => - wikiMap.get(cgToWikiMap.get(id) || id) || - wikiMap.get( - urlToIdMap.get(`https://www.coingecko.com/en/${coinType}/${id}`) || '', + const wikisByUrl = await wikiQuery + .where('wiki.hidden = false') + .andWhere( + `EXISTS ( + SELECT 1 + FROM json_array_elements(wiki.metadata) AS meta + WHERE meta->>'id' = 'coingecko_profile' + AND meta->>'value' IN (:...urls) + )`, + { urls: urlsToSearch }, + ) + .getMany() + + const urlToWikiMap = new Map() + for (const wiki of wikisByUrl) { + const coingeckoMeta = wiki.metadata?.find( + (meta: any) => meta.id === 'coingecko_profile', ) + if (coingeckoMeta?.value) { + const coingeckoId = coingeckoMeta.value.split('/').pop() + if (coingeckoId) { + urlToWikiMap.set(coingeckoId, wiki) + } + } + } - const allLinkedIds = new Set() - const wikiIdsForEvents: string[] = [] - uncachedIds.forEach((id) => { - const w = getWiki(id) - if (w) { - wikiIdsForEvents.push(w.id) - w.linkedWikis?.founders?.forEach((f: string) => allLinkedIds.add(f)) - w.linkedWikis?.blockchains?.forEach((b: string) => allLinkedIds.add(b)) + for (const item of notFoundItems) { + const foundWiki = urlToWikiMap.get(item.coingeckoId) + if (foundWiki) { + const existingIndex = wikiResults.findIndex( + (r) => r.index === item.index, + ) + if (existingIndex !== -1) { + wikiResults[existingIndex].wiki = foundWiki + } } - }) + } + } - const [linkedWikis, events] = await Promise.all([ - allLinkedIds.size - ? (this.dataSource.query( - `SELECT id, title, ipfs, images FROM wiki WHERE id = ANY($1::text[]) AND hidden = false`, - [[...allLinkedIds]], - ) as Promise) - : Promise.resolve([]), - wikiIdsForEvents.length - ? (this.dataSource.query( - `SELECT * FROM events WHERE "wikiId" = ANY($1::text[])`, - [wikiIdsForEvents], - ) as Promise) - : Promise.resolve([]), - ]) + private async enrichWikisWithRelatedData( + wikiResults: { index: number; wiki: Wiki | null }[], + ): Promise> { + const { allFounderIds, allBlockchainIds, allWikiIds } = + this.collectRelatedIds(wikiResults) - const linkedMap = new Map( - linkedWikis.map((w: LinkedWikiData) => [w.id, w]), - ) - const eventsMap = events.reduce( - (acc: Map, e: EventData) => { - acc.set(e.wikiId, [...(acc.get(e.wikiId) || []), e]) - return acc - }, - new Map(), + const [foundersMap, blockchainsMap, eventsMap] = + await this.fetchRelatedData(allFounderIds, allBlockchainIds, allWikiIds) + + return this.buildEnrichedResults( + wikiResults, + foundersMap, + blockchainsMap, + eventsMap, ) + } - for (const id of uncachedIds) { - const wikiData = getWiki(id) - const result: RankPageWiki = wikiData - ? { - wiki: { - ...wikiData, - events: eventsMap.get(wikiData.id) || [], - tags: wikiData.tags, - } as unknown as Wiki, - founders: (wikiData.linkedWikis?.founders || []) - .map((f: string) => linkedMap.get(f)) - .filter( - (wiki): wiki is LinkedWikiData => wiki !== undefined, - ) as unknown as Wiki[], - blockchain: (wikiData.linkedWikis?.blockchains || []) - .map((b: string) => linkedMap.get(b)) - .filter( - (wiki): wiki is LinkedWikiData => wiki !== undefined, - ) as unknown as Wiki[], - } - : { wiki: null as any, founders: [], blockchain: [] } + private collectRelatedIds( + wikiResults: { index: number; wiki: Wiki | null }[], + ) { + const allFounderIds = new Set() + const allBlockchainIds = new Set() + const allWikiIds = new Set() + + for (const { wiki } of wikiResults) { + if (wiki?.id) { + allWikiIds.add(wiki.id) + wiki.linkedWikis?.founders?.forEach((id) => allFounderIds.add(id)) + wiki.linkedWikis?.blockchains?.forEach((id) => allBlockchainIds.add(id)) + } + } + + return { allFounderIds, allBlockchainIds, allWikiIds } + } + + private async fetchRelatedData( + allFounderIds: Set, + allBlockchainIds: Set, + allWikiIds: Set, + ): Promise<[Map, Map, Map]> { + const wikiRepository = this.dataSource.getRepository(Wiki) + const eventsRepository = this.dataSource.getRepository(Events) + + const baseQuery = wikiRepository + .createQueryBuilder('wiki') + .select([ + 'wiki.id', + 'wiki.title', + 'wiki.ipfs', + 'wiki.metadata', + 'wiki.images', + ]) + + const [foundersArray, blockchainsArray, eventsArray] = await Promise.all([ + allFounderIds.size > 0 + ? baseQuery + .clone() + .where('wiki.id IN (:...ids) AND wiki.hidden = false', { + ids: Array.from(allFounderIds), + }) + .getMany() + : [], + allBlockchainIds.size > 0 + ? baseQuery + .clone() + .where('wiki.id IN (:...ids) AND wiki.hidden = false', { + ids: Array.from(allBlockchainIds), + }) + .getMany() + : [], + allWikiIds.size > 0 + ? eventsRepository.query( + `SELECT * FROM events WHERE "wikiId" = ANY($1)`, + [Array.from(allWikiIds)], + ) + : [], + ]) - if (wikiData && this.INCOMING_WIKI_ID === wikiData.id) - this.CACHED_WIKI = result - resultsMap.set(id, result) - this.cacheManager.set(id, result, 3600 * 1000) + const foundersMap = new Map(foundersArray.map((f) => [f.id, f])) + const blockchainsMap = new Map(blockchainsArray.map((b) => [b.id, b])) + const eventsMap = new Map() + + for (const event of eventsArray) { + if (!eventsMap.has(event.wikiId)) { + eventsMap.set(event.wikiId, []) + } + eventsMap.get(event.wikiId)!.push(event) + } + + return [foundersMap, blockchainsMap, eventsMap] + } + + private buildEnrichedResults( + wikiResults: { index: number; wiki: Wiki | null }[], + foundersMap: Map, + blockchainsMap: Map, + eventsMap: Map, + ): Map { + const resultsMap = new Map() + + for (const { index, wiki } of wikiResults) { + if (!wiki) { + resultsMap.set(index, null) + continue + } + + const founders = + wiki.linkedWikis?.founders + ?.map((id) => foundersMap.get(id)) + .filter(Boolean) || [] + + const blockchain = + wiki.linkedWikis?.blockchains + ?.map((id) => blockchainsMap.get(id)) + .filter(Boolean) || [] + + const events = eventsMap.get(wiki.id) || [] + const wikiWithEvents = { ...wiki, events } + + resultsMap.set(index, { + wiki: wikiWithEvents, + founders, + blockchain, + } as RankPageWiki) } return resultsMap } + private async cacheResults( + missingItems: { index: number; id: string }[], + resultsMap: Map, + ): Promise { + for (const item of missingItems) { + const rankPageWiki = resultsMap.get(item.index) + if (rankPageWiki) { + await this.cacheManager.set(item.id, rankPageWiki) + } + } + } + + private mergeResults( + items: { index: number; id: string; rankPageWiki: RankPageWiki | null }[], + resultsMap: Map, + ): { index: number; id: string; rankPageWiki: RankPageWiki | null }[] { + return items.map((item) => ({ + ...item, + rankPageWiki: resultsMap.get(item.index) ?? item.rankPageWiki, + })) + } + async getWikiData( coinsData: Record | undefined, kind: RankType, delay = false, - ): Promise<(RankPageWiki | null)[]> { - if (!coinsData?.length) return [] + ): Promise { + const k = kind.toLowerCase() const batchSize = 100 const allWikis: (RankPageWiki | null)[] = [] - for (let i = 0; i < coinsData.length; i += batchSize) { - const batch = coinsData.slice(i, i + batchSize) - const wikisMap = await this.findWikisBulk( - batch.map((e: any) => e.id), - kind.toLowerCase(), - ) - - allWikis.push(...batch.map((e: any) => wikisMap.get(e.id) || null)) - - if (delay) { - await new Promise((r) => setTimeout(r, 2000)) + if (coinsData) { + for (let i = 0; i < coinsData.length; i += batchSize) { + const batch = coinsData.slice(i, i + batchSize) + const batchItems = await this.findWikis( + k, + batch.map((element: any, index: number) => ({ + index, + id: element.id, + rankPageWiki: null, + })), + ) + const sortedItems = batchItems.sort((a, b) => a.index - b.index) + allWikis.push(...sortedItems.map((item) => item.rankPageWiki)) + if (delay) { + await new Promise((r) => setTimeout(r, 2000)) + } } } - - return allWikis + return allWikis as RankPageWiki[] } async *marketData(args: MarketCapInputs, reset = false) { @@ -284,7 +454,6 @@ class MarketCapService { return this.processMarketElement(element, wiki, kind) }), ) - yield processedResults } } @@ -323,7 +492,6 @@ class MarketCapService { floor_price_in_usd_24h_percentage_change: element.floor_price_in_usd_24h_percentage_change || 0, } - const marketData = { [kind === RankType.TOKEN ? 'tokenMarketData' : 'nftMarketData']: { hasWiki: !!rankpageWiki?.wiki, @@ -344,7 +512,7 @@ class MarketCapService { id: rankpageWiki.wiki.id, title: rankpageWiki.wiki.title, images: rankpageWiki.wiki.images, - tags: rankpageWiki.wiki.tags?.map((t: any) => ({ id: t.id })), + tags: rankpageWiki.wiki.__tags__?.map((t: any) => ({ id: t.id })), events: rankpageWiki.wiki.events?.map((e: any) => ({ type: e.type, date: e.date, diff --git a/src/App/marketCap/marketcap.dto.ts b/src/App/marketCap/marketcap.dto.ts index 3b07e073..a6ff7678 100644 --- a/src/App/marketCap/marketcap.dto.ts +++ b/src/App/marketCap/marketcap.dto.ts @@ -178,3 +178,4 @@ export const STABLECOIN_CATEGORIES_CACHE_KEY = 'stablecoinCategories' export const NO_WIKI_MARKETCAP_SEARCH_CACHE_KEY = 'noWikiMarketcapSearch' export const MARKETCAP_SEARCH_CACHE_KEY = 'marketcapSearch' export const BASE_URL_COINGECKO_API = 'https://pro-api.coingecko.com/api/v3/' +export const baseCoingeckoUrl = 'https://www.coingecko.com/en' diff --git a/src/App/pinJSONAndImage/pin.service.ts b/src/App/pinJSONAndImage/pin.service.ts index 88c95c2f..521f9df7 100644 --- a/src/App/pinJSONAndImage/pin.service.ts +++ b/src/App/pinJSONAndImage/pin.service.ts @@ -533,13 +533,18 @@ class PinService { `https://www.coingecko.com/en/coins/${apiId}` this.logger.debug('wiki id', wiki.id, '🔗', 'coingecko api Id', apiId) const saveMatchedIdcallback = async () => { - const matchedId = marketCapIdRepo.create({ - wikiId: wiki.id, - coingeckoId: apiId, - kind: RankType.TOKEN, - linked: false, - }) - await marketCapIdRepo.save(matchedId) + await marketCapIdRepo.upsert( + { + wikiId: wiki.id, + coingeckoId: apiId, + kind: RankType.TOKEN, + linked: false, + }, + { + conflictPaths: ['wikiId', 'coingeckoId'], + skipUpdateIfNoValuesChanged: true, + }, + ) } return { diff --git a/src/Database/Entities/marketCapIds.entity.ts b/src/Database/Entities/marketCapIds.entity.ts index 751f6687..6aecc7fd 100644 --- a/src/Database/Entities/marketCapIds.entity.ts +++ b/src/Database/Entities/marketCapIds.entity.ts @@ -8,10 +8,10 @@ class MarketCapIds { @PrimaryGeneratedColumn('uuid') id!: string - @Column('varchar') + @Column({ type: 'varchar', unique: true }) wikiId!: string - @Column({ type: 'varchar' }) + @Column({ type: 'varchar', unique: true }) coingeckoId!: string @Column({