GeezerVison is showing github.com — aicodingstack.io/src/lib/model-intelligence-index.ts at main · aicodingstack/aicodingstack.io

Expand file tree

model-intelligence-index.ts

Copy path

More file actions

Latest commit

History

209 lines (179 loc) · 5.54 KB

model-intelligence-index.ts

Copy path

File metadata and controls

209 lines (179 loc) · 5.54 KB

Raw

Copy raw file

Download raw file

Open symbols panel

Edit and raw actions

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

147

148

149

150

151

152

153

154

155

156

157

158

159

160

161

162

163

164

165

166

167

168

169

170

171

172

173

174

175

176

177

178

179

180

181

182

183

184

185

186

187

188

189

190

191

192

193

194

195

196

197

198

199

200

201

202

203

204

205

206

207

208

209

import { modelsData } from '@/lib/generated/models'

import { vendorsData } from '@/lib/generated/vendors'

import { findVendorByName } from '@/lib/vendor-identity'

import artificialAnalysisData from '../../data/artificial-analysis-index.json'

export interface ArtificialAnalysisIndexEntry {

modelId: string

score: number

estimated: boolean

configuration: string

}

export interface ArtificialAnalysisIndexData {

$schema: './$schemas/artificial-analysis-index.schema.json'

source: 'Artificial Analysis'

sourceUrl: string

methodologyUrl: string

indexVersion: string

observedAt: string

legacyMissingModelIds: string[]

entries: ArtificialAnalysisIndexEntry[]

}

const artificialAnalysisIndex = artificialAnalysisData as ArtificialAnalysisIndexData

const FALLBACK_COLOR: ModelIntelligenceThemeColor = {

light: '#6b7280',

dark: '#9ca3af',

}

export const modelIntelligenceLegacyMissingModelIds = artificialAnalysisIndex.legacyMissingModelIds

export interface ModelIntelligenceThemeColor {

light: string

dark: string

}

export type ModelIntelligenceMarker =

| 'circle'

| 'square'

| 'triangle'

| 'diamond'

| 'cross'

| 'star'

| 'wye'

export interface ModelIntelligencePoint {

modelId: string

vendor: string

series: string

seriesId: string

seriesOrder: number

score: number

estimated: boolean

configuration: string

releaseDate: string

timestamp: number

color: ModelIntelligenceThemeColor

}

export interface ModelIntelligenceSeries {

id: string

vendor: string

order: number

color: ModelIntelligenceThemeColor

dash: string | null

marker: ModelIntelligenceMarker

points: ModelIntelligencePoint[]

}

export function createTimelineTicks([start, end]: [number, number], tickCount = 7): number[] {

if (!Number.isFinite(start) || !Number.isFinite(end) || tickCount < 1) return []

if (start === end || tickCount === 1) return [start]

const count = Math.max(2, Math.floor(tickCount))

const ticks = Array.from({ length: count }, (_, index) =>

index === count - 1 ? end : Math.round(start + ((end - start) * index) / (count - 1))

)

return ticks.filter((tick, index) => index === 0 || tick !== ticks[index - 1])

}

const modelById = new Map(modelsData.map(model => [model.id, model]))

const modelSeriesByModelId = new Map<

string,

{

vendorId: string

seriesId: string

seriesName: string

seriesOrder: number

}

>()

for (const vendor of vendorsData) {

vendor.modelSeries?.forEach((series, seriesOrder) => {

for (const modelId of series.modelIds) {

if (modelSeriesByModelId.has(modelId)) {

throw new Error(`Model belongs to multiple vendor series: ${modelId}`)

}

modelSeriesByModelId.set(modelId, {

vendorId: vendor.id,

seriesId: series.id,

seriesName: series.name,

seriesOrder,

})

}

})

}

export const allModelIntelligencePoints: ModelIntelligencePoint[] =

artificialAnalysisIndex.entries.map(entry => {

const model = modelById.get(entry.modelId)

if (!model?.releaseDate) {

throw new Error(`Artificial Analysis entry has no matching dated model: ${entry.modelId}`)

}

const series = modelSeriesByModelId.get(model.id)

if (!series) {

throw new Error(`Artificial Analysis model has no vendor series: ${model.id}`)

}

const vendor = findVendorByName(vendorsData, model.vendor)

if (!vendor || series.vendorId !== vendor.id) {

throw new Error(`Model series vendor does not match model vendor: ${model.id}`)

}

return {

...entry,

vendor: model.vendor,

series: series.seriesName,

seriesId: series.seriesId,

seriesOrder: series.seriesOrder,

releaseDate: model.releaseDate,

timestamp: Date.parse(`${model.releaseDate}T00:00:00Z`),

color: vendor.themeColor ?? FALLBACK_COLOR,

}

})

const groupedSeries = new Map<string, ModelIntelligenceSeries>()

for (const point of allModelIntelligencePoints) {

const id = `${point.vendor}:${point.seriesId}`

const existing = groupedSeries.get(id)

if (existing) {

existing.points.push(point)

} else {

groupedSeries.set(id, {

id,

vendor: point.vendor,

order: point.seriesOrder,

color: point.color,

dash: null,

marker: 'circle',

points: [point],

})

}

const sortedSeries = Array.from(groupedSeries.values())

.map(series => ({

...series,

points: series.points.sort((a, b) => a.timestamp - b.timestamp || a.score - b.score),

}))

.sort(

(a, b) =>

a.vendor.localeCompare(b.vendor) ||

a.order - b.order ||

a.name.localeCompare(b.name, 'en', { numeric: true, sensitivity: 'base' })

)

const seriesIndexByVendor = new Map<string, number>()

const DASH_PATTERNS = [null, '6 4', '2 4', '10 4 2 4']

const MARKERS: ModelIntelligenceMarker[] = [

'circle',

'square',

'triangle',

'diamond',

'cross',

'star',

'wye',

]

export const modelIntelligenceSeries = sortedSeries.map(series => {

const seriesIndex = seriesIndexByVendor.get(series.vendor) ?? 0

seriesIndexByVendor.set(series.vendor, seriesIndex + 1)

return {

...series,

dash: DASH_PATTERNS[seriesIndex % DASH_PATTERNS.length] ?? null,

marker: MARKERS[seriesIndex % MARKERS.length] ?? 'circle',

}

})

export const modelIntelligenceMeta = {

source: artificialAnalysisIndex.source,

sourceUrl: artificialAnalysisIndex.sourceUrl,

methodologyUrl: artificialAnalysisIndex.methodologyUrl,

indexVersion: artificialAnalysisIndex.indexVersion,

observedAt: artificialAnalysisIndex.observedAt,

}