Skip to content

Commit

Permalink
fix: metadata missing for some assets (#1443)
Browse files Browse the repository at this point in the history 
* debug metadata

* more logs

* add log

* wrap worker operations in a transaction

* lint

* read uncommited
  • Loading branch information
@justraman
justraman authored Dec 4, 2024
1 parent bddc64f commit 622bcd8
Show file tree
Hide file tree
Showing 10 changed files with 305 additions and 383 deletions.
101 changes: 51 additions & 50 deletions src/job-handlers/collection-stats.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,60 +15,61 @@ export default async (job: Queue.Job<JobData>, done: Queue.DoneCallback) => {
throw err
})
}
const { collectionId } = job.data
const em = connection.manager

const promises = [
em
.createQueryBuilder()
.addSelect('MAX(l.highest_price) AS highest_sale')
.addSelect('MAX(l.last_sale) AS last_sale')
.addSelect('SUM(l.volume) AS total_volume')
.addSelect('SUM(l.count)::int AS sales')
.from((qb) => {
return qb
.select('listing.id AS id')
.addSelect('COUNT(sale.id) AS count')
.addSelect('listing.highest_price AS highest_price')
.addSelect('SUM(sale.amount * sale.price) AS volume')
.addSelect('listing.amount AS amount')
.addSelect(
'CASE WHEN lead(listing.id) OVER(ORDER BY listing.created_at) IS NULL THEN listing.highest_price END AS last_sale'
)
.from(ListingSale, 'sale')
.innerJoin(Listing, 'listing', 'listing.id = sale.listing')
.innerJoin(Token, 'token', 'listing.make_asset_id_id = token.id')
.innerJoin(Collection, 'collection', 'token.collection = collection.id')
.leftJoin(ListingStatus, 'status', `listing.id = status.listing AND status.type = 'Finalized'`)
.where('collection.id = :collectionId', { collectionId })
.groupBy('listing.id')
}, 'l')
.getRawOne(),
const { collectionId } = job.data
connection.manager.transaction('READ UNCOMMITTED', async (em) => {
const promises = [
em
.createQueryBuilder()
.addSelect('MAX(l.highest_price) AS highest_sale')
.addSelect('MAX(l.last_sale) AS last_sale')
.addSelect('SUM(l.volume) AS total_volume')
.addSelect('SUM(l.count)::int AS sales')
.from((qb) => {
return qb
.select('listing.id AS id')
.addSelect('COUNT(sale.id) AS count')
.addSelect('listing.highest_price AS highest_price')
.addSelect('SUM(sale.amount * sale.price) AS volume')
.addSelect('listing.amount AS amount')
.addSelect(
'CASE WHEN lead(listing.id) OVER(ORDER BY listing.created_at) IS NULL THEN listing.highest_price END AS last_sale'
)
.from(ListingSale, 'sale')
.innerJoin(Listing, 'listing', 'listing.id = sale.listing')
.innerJoin(Token, 'token', 'listing.make_asset_id_id = token.id')
.innerJoin(Collection, 'collection', 'token.collection = collection.id')
.leftJoin(ListingStatus, 'status', `listing.id = status.listing AND status.type = 'Finalized'`)
.where('collection.id = :collectionId', { collectionId })
.groupBy('listing.id')
}, 'l')
.getRawOne(),

em.query(floorQuery, [collectionId]),
em
.getRepository(Token)
.createQueryBuilder('token')
.select('COUNT(token.id)', 'tokenCount')
.addSelect('SUM(token.supply)', 'supply')
.where('token.collection = :collectionId', { collectionId })
.getRawOne(),
]
em.query(floorQuery, [collectionId]),
em
.getRepository(Token)
.createQueryBuilder('token')
.select('COUNT(token.id)', 'tokenCount')
.addSelect('SUM(token.supply)', 'supply')
.where('token.collection = :collectionId', { collectionId })
.getRawOne(),
]

const [sales, [{ floor_price }], { tokenCount, supply }] = await Promise.all(promises)
const [sales, [{ floor_price }], { tokenCount, supply }] = await Promise.all(promises)

const stats = new CollectionStats({
tokenCount: Number(tokenCount),
supply: BigInt(supply ?? 0n),
salesCount: sales?.sales ?? 0,
volume: sales?.total_volume ?? 0n,
marketCap: BigInt(sales?.last_sale ?? 0n) * BigInt(tokenCount),
floorPrice: floor_price,
lastSale: sales?.last_sale ?? null,
highestSale: sales?.highest_sale ?? null,
})
const stats = new CollectionStats({
tokenCount: Number(tokenCount),
supply: BigInt(supply ?? 0n),
salesCount: sales?.sales ?? 0,
volume: sales?.total_volume ?? 0n,
marketCap: BigInt(sales?.last_sale ?? 0n) * BigInt(tokenCount),
floorPrice: floor_price,
lastSale: sales?.last_sale ?? null,
highestSale: sales?.highest_sale ?? null,
})

await em.update(Collection, { id: collectionId }, { stats })
await em.update(Collection, { id: collectionId }, { stats })

done(null, { id: collectionId, stats: stats.toJSON() })
done(null, { id: collectionId, stats: stats.toJSON() })
})
}
186 changes: 93 additions & 93 deletions src/job-handlers/compute-traits.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,105 +23,105 @@ export default async (job: Queue.Job<JobData>, done: Queue.DoneCallback) => {
})
}

const em = connection.manager

const traitTypeMap = new Map<string, TraitValueMap>()
const tokenTraitMap = new Map<string, string[]>()

const start = new Date()

const { collectionId } = job.data satisfies JobData

const tokens = await em
.getRepository(Token)
.createQueryBuilder('token')
.select('token.id')
.addSelect('token.metadata')
.addSelect('token.supply')
.where('token.collection = :collectionId', { collectionId })
.andWhere('token.supply > 0')
.getMany()

await em.query(`DELETE FROM trait_token USING trait WHERE trait.id = trait_token.trait_id AND trait.collection_id = $1`, [
collectionId,
])

await em.query(`DELETE FROM trait WHERE collection_id = $1`, [collectionId])

tokens.forEach((token) => {
if (!token.metadata || !token.metadata.attributes || !isPlainObject(token.metadata.attributes)) return
const attributes = token.metadata.attributes as Record<string, { value: string } | string>
Object.entries(attributes).forEach(([traitType, data]) => {
let value = data as string
if (typeof data === 'object') {
value = data.value
}

if (!value) return

value = value.toString()

if (!traitTypeMap.has(traitType)) {
traitTypeMap.set(traitType, new Map())
}
const tType = traitTypeMap.get(traitType) as TraitValueMap
if (tType.has(value)) {
tType.set(value, (tType.get(value) as bigint) + token.supply)
} else {
tType.set(value, token.supply)
}

tokenTraitMap.set(token.id, [...(tokenTraitMap.get(token.id) || []), `${collectionId}-${traitType}-${value}`])
connection.manager.transaction('READ UNCOMMITTED', async (em) => {
const traitTypeMap = new Map<string, TraitValueMap>()
const tokenTraitMap = new Map<string, string[]>()

const start = new Date()

const { collectionId } = job.data satisfies JobData

const tokens = await em
.getRepository(Token)
.createQueryBuilder('token')
.select('token.id')
.addSelect('token.metadata')
.addSelect('token.supply')
.where('token.collection = :collectionId', { collectionId })
.andWhere('token.supply > 0')
.getMany()

await em.query(`DELETE FROM trait_token USING trait WHERE trait.id = trait_token.trait_id AND trait.collection_id = $1`, [
collectionId,
])

await em.query(`DELETE FROM trait WHERE collection_id = $1`, [collectionId])

tokens.forEach((token) => {
if (!token.metadata || !token.metadata.attributes || !isPlainObject(token.metadata.attributes)) return
const attributes = token.metadata.attributes as Record<string, { value: string } | string>
Object.entries(attributes).forEach(([traitType, data]) => {
let value = data as string
if (typeof data === 'object') {
value = data.value
}

if (!value) return

value = value.toString()

if (!traitTypeMap.has(traitType)) {
traitTypeMap.set(traitType, new Map())
}
const tType = traitTypeMap.get(traitType) as TraitValueMap
if (tType.has(value)) {
tType.set(value, (tType.get(value) as bigint) + token.supply)
} else {
tType.set(value, token.supply)
}

tokenTraitMap.set(token.id, [...(tokenTraitMap.get(token.id) || []), `${collectionId}-${traitType}-${value}`])
})
})
})

if (!traitTypeMap.size) {
job.log(`No traits found for collection ${collectionId}`)
done()

return
}

job.log(`Found ${traitTypeMap.size} trait types`)
const traitsToSave: Trait[] = []

traitTypeMap.forEach((traitValueMap, traitType) => {
traitValueMap.forEach((count, value) => {
traitsToSave.push(
new Trait({
id: hash(`${collectionId}-${traitType}-${value}`),
collection: new Collection({ id: collectionId }),
traitType,
value,
count,
})
)
if (!traitTypeMap.size) {
job.log(`No traits found for collection ${collectionId}`)
done()

return
}

job.log(`Found ${traitTypeMap.size} trait types`)
const traitsToSave: Trait[] = []

traitTypeMap.forEach((traitValueMap, traitType) => {
traitValueMap.forEach((count, value) => {
traitsToSave.push(
new Trait({
id: hash(`${collectionId}-${traitType}-${value}`),
collection: new Collection({ id: collectionId }),
traitType,
value,
count,
})
)
})
})
})

job.log(`Saving ${traitsToSave.length} traits`)
await em.save(Trait, traitsToSave as any, { chunk: 1000 })
const traitTokensToSave: TraitToken[] = []

tokenTraitMap.forEach((traits, tokenId) => {
if (!traits.length) return
traits.forEach((trait) => {
traitTokensToSave.push(
new TraitToken({
id: hash(`${trait}-${tokenId}`),
trait: new Trait({ id: hash(trait) }),
token: new Token({ id: tokenId }),
})
)
job.log(`Saving ${traitsToSave.length} traits`)
await em.save(Trait, traitsToSave as any, { chunk: 1000 })
const traitTokensToSave: TraitToken[] = []

tokenTraitMap.forEach((traits, tokenId) => {
if (!traits.length) return
traits.forEach((trait) => {
traitTokensToSave.push(
new TraitToken({
id: hash(`${trait}-${tokenId}`),
trait: new Trait({ id: hash(trait) }),
token: new Token({ id: tokenId }),
})
)
})
})
})

if (traitTokensToSave.length) {
job.log(`Saving ${traitsToSave.length} token traits`)
await em.save(TraitToken, traitTokensToSave as any, { chunk: 1000 })
}
if (traitTokensToSave.length) {
job.log(`Saving ${traitsToSave.length} token traits`)
await em.save(TraitToken, traitTokensToSave as any, { chunk: 1000 })
}

computeRarityRank(collectionId)
computeRarityRank(collectionId)

done(null, { timeElapsed: new Date().getTime() - start.getTime() })
done(null, { timeElapsed: new Date().getTime() - start.getTime() })
})
}
29 changes: 0 additions & 29 deletions src/job-handlers/delete-traits.ts
View file

This file was deleted.

Loading

0 comments on commit 622bcd8

Please sign in to comment.