fix(manifest): restore start block and substreams defaults

README.md

@@ -57,10 +57,19 @@ Everytime a new release is merged into `master` there will be a new github relea
 # Deploying the subgraph
 
 The deployment manifest IPFS handler uses the native YAML API from `graph-ts` 0.38.2
-and requires **Graph Node 0.37.0 or newer**. Malformed manifests retain their raw
-content; derived fields that cannot be safely extracted remain unset and produce
-warnings. Schema links support CIDv0 and CIDv1 in base32 or base58btc, optionally
-with an IPFS file path. Unsupported links are skipped without creating a file data source.
+and requires **Graph Node 0.40.2 or newer** to resolve schema references directly,
+including IPFS URIs and gateway/RPC URLs. Malformed manifests retain their raw
+content. `startBlock` defaults to `0` and `poweredBySubstreams` to `false` when they
+cannot be determined; missing or invalidly typed network/schema fields remain unset.
+Extracted network and schema-reference strings are trimmed, bounded in length, and
+checked for embedded control characters. Schema references are otherwise treated as
+opaque metadata: Graph Node resolves the original reference and logs/skips invalid
+IPFS sources. Common IPFS prefixes and simple gateway paths are normalized only for
+metadata and entity IDs; URL query/fragment syntax is retained. A schema reference
+can be present even if the schema file cannot be resolved. Schema entity IDs over
+2048 UTF-8 bytes are hashed to stay within database index limits; reference metadata
+is retained. Consumers must escape
+metadata appropriately when displaying text or constructing links.
 
 The npm scripts are set up to deploy the subgraphs in one command. Mainnet is connected to a hook
 where it will be deployed automatically when the `master` branch is updated. Therefore, we never

src/mappings/helpers/manifest.ts

@@ -32,97 +32,40 @@ export function manifestStartBlock(value: YAMLValue): BigInt | null {
   return BigInt.fromString(text)
 }
 
-// Decode the CID encodings normally used in published manifests. Unsupported
-// encodings are skipped rather than passed to a host function that can abort.
-function decodeCid(text: string): Uint8Array | null {
-  if (text.length == 0 || text.length > 128) return null
-  let base58 = text.startsWith('Qm') || text.startsWith('z')
-  if (base58) {
-    let encoded = text.startsWith('z') ? text.slice(1) : text
-    let alphabet = '123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz'
-    let bytes = new Array<u8>()
-    for (let i = 0; i < encoded.length; i++) {
-      let carry = alphabet.indexOf(encoded.charAt(i))
-      if (carry < 0) return null
-      for (let j = 0; j < bytes.length; j++) {
-        carry += i32(bytes[j]) * 58
-        bytes[j] = u8(carry & 255)
-        carry >>= 8
-      }
-      while (carry > 0) {
-        bytes.push(u8(carry & 255))
-        carry >>= 8
-      }
-    }
-    for (let i = 0; i < encoded.length && encoded.charAt(i) == '1'; i++) bytes.push(0)
-    let result = new Uint8Array(bytes.length)
-    for (let i = 0; i < bytes.length; i++) result[i] = bytes[bytes.length - i - 1]
-    return result
-  }
-  if (!text.startsWith('b') && !text.startsWith('B')) return null
-  let alphabet = text.startsWith('b') ? 'abcdefghijklmnopqrstuvwxyz234567' : 'ABCDEFGHIJKLMNOPQRSTUVWXYZ234567'
-  let result = new Uint8Array((text.length - 1) * 5 / 8)
-  let bits = 0
-  let buffer = 0
-  let offset = 0
-  for (let i = 1; i < text.length; i++) {
-    let digit = alphabet.indexOf(text.charAt(i))
-    if (digit < 0) return null
-    buffer = (buffer << 5) | digit
-    bits += 5
-    if (bits >= 8) {
-      bits -= 8
-      result[offset++] = u8(buffer >> bits)
-      buffer &= (1 << bits) - 1
-    }
-  }
-  return bits < 5 && buffer == 0 ? result : null
-}
-
-function validCid(text: string): bool {
-  let bytes = decodeCid(text)
-  if (bytes === null) return false
-  if (text.startsWith('Qm')) {
-    return text.length == 46 && bytes.length == 34 && bytes[0] == 0x12 && bytes[1] == 0x20
-  }
-  // CIDv1 contains four unsigned varints: version, codec, hash code, digest
-  // length; followed by the digest. Graph Node supports digests up to 64 bytes.
-  let offset = 0
-  for (let field = 0; field < 4; field++) {
-    let value: u64 = 0
-    let terminated = false
-    for (let i = 0; i < 10 && offset < bytes.length; i++) {
-      let byte = bytes[offset++]
-      if (i == 9 && byte > 1) return false
-      value |= u64(byte & 127) << (i * 7)
-      if ((byte & 128) == 0) {
-        if (i > 0 && byte == 0) return false
-        terminated = true
-        break
-      }
-    }
-    if (!terminated) return false
-    if (field == 0 && value != 1) return false
-    if (field == 3) return value <= 64 && value == u64(bytes.length - offset)
+// Metadata is opaque text, not HTML or an executable URL. Keep basic bounds
+// and reject embedded control characters without imposing format whitelists.
+export function manifestMetadataString(value: YAMLValue | null, maxLength: i32): string | null {
+  let text = yamlString(value)
+  if (text === null || text.length > maxLength) return null
+  for (let i = 0; i < text.length; i++) {
+    let code = text.charCodeAt(i)
+    if (code < 32 || (code >= 127 && code <= 159)) return null
   }
-  return false
+  return text
 }
 
-export function manifestSchemaPath(value: YAMLValue | null): string | null {
+export function manifestSchemaReference(value: YAMLValue | null): string | null {
   // Published manifests use { '/': '/ipfs/CID' }; also accept string links.
   if (value !== null && value.isObject()) value = yamlField(value, '/')
-  let path = yamlString(value)
-  if (path === null || path.length > 2048) return null
+  return manifestMetadataString(value, 2048)
+}
+
+// Normalize familiar references for stable metadata and entity IDs. This is
+// not validation: Graph Node receives the original reference for resolution.
+export function manifestSchemaPath(reference: string): string {
+  let path = reference
+  if (path.startsWith('http://') || path.startsWith('https://')) {
+    // Keep URL-specific syntax intact instead of reimplementing a URL parser.
+    if (path.includes('?') || path.includes('#')) return reference
+    let authorityStart = path.startsWith('https://') ? 8 : 7
+    let pathStart = path.indexOf('/', authorityStart)
+    if (pathStart <= authorityStart) return reference
+    let ipfsStart = path.indexOf('/ipfs/', pathStart)
+    if (ipfsStart < 0) return reference
+    path = path.slice(ipfsStart + 6)
+  }
   if (path.startsWith('/ipfs/')) path = path.slice(6)
   else if (path.startsWith('ipfs://')) path = path.slice(7)
-  let segments = path.split('/')
-  if (!validCid(segments[0])) return null
-  for (let i = 1; i < segments.length; i++) {
-    if (segments[i].length == 0 || segments[i] == '.' || segments[i] == '..') return null
-  }
-  for (let i = 0; i < path.length; i++) {
-    let code = path.charCodeAt(i)
-    if (code <= 32 || code == 127 || path.charAt(i) == '?' || path.charAt(i) == '#' || path.charAt(i) == '%' || path.charAt(i) == '\\') return null
-  }
-  return path
+  while (path.endsWith('/')) path = path.slice(0, path.length - 1)
+  return path.length > 0 ? path : reference
 }

src/mappings/ipfs.ts

@@ -1,4 +1,4 @@
-import { json, Bytes, dataSource, JSONValueKind, log, DataSourceContext, BigInt, yaml, YAMLValue } from '@graphprotocol/graph-ts'
+import { json, Bytes, dataSource, JSONValueKind, log, DataSourceContext, BigInt, yaml, YAMLValue, crypto } from '@graphprotocol/graph-ts'
 import {
   SubgraphMeta,
   SubgraphVersionMeta,
@@ -10,7 +10,7 @@ import {
   SubgraphDeploymentSchema as SubgraphDeploymentSchemaTemplate
 } from '../types/templates'
 import { jsonToString } from './utils'
-import { yamlField, yamlString, manifestStartBlock, manifestSchemaPath } from './helpers/manifest'
+import { yamlField, yamlString, manifestStartBlock, manifestMetadataString, manifestSchemaReference, manifestSchemaPath } from './helpers/manifest'
 
 export function handleSubgraphMetadata(content: Bytes): void {
   let id = dataSource.context().getString("id")
@@ -86,17 +86,23 @@ function manifestWarning(id: string, field: string): void {
 }
 
 function readManifestSchema(manifest: SubgraphDeploymentManifest, root: YAMLValue): void {
-  let path = manifestSchemaPath(yamlField(yamlField(root, 'schema'), 'file'))
-  if (path === null) {
+  let reference = manifestSchemaReference(yamlField(yamlField(root, 'schema'), 'file'))
+  if (reference === null) {
     manifestWarning(manifest.id, 'schema.file')
     return
   }
+  let path = manifestSchemaPath(reference)
   let schemaId = manifest.id.concat('-').concat(path)
+  // Immutable entity IDs are indexed. Preserve ordinary IDs while bounding
+  // their UTF-8 size even when references contain multibyte characters.
+  let schemaIdBytes = Bytes.fromUTF8(schemaId)
+  if (schemaIdBytes.length > 2048) schemaId = crypto.keccak256(schemaIdBytes).toHexString()
   manifest.schema = schemaId
   manifest.schemaIpfsHash = path
   let context = new DataSourceContext()
   context.setString('id', schemaId)
-  SubgraphDeploymentSchemaTemplate.createWithContext(path, context)
+  // Graph Node validates the original reference and skips invalid IPFS sources.
+  SubgraphDeploymentSchemaTemplate.createWithContext(reference, context)
 }
 
 function readManifestNetwork(manifest: SubgraphDeploymentManifest, root: YAMLValue): void {
@@ -107,8 +113,8 @@ function readManifestNetwork(manifest: SubgraphDeploymentManifest, root: YAMLVal
     if (entries === null || !entries.isArray()) continue
     let sources = entries.toArray()
     for (let i = 0; i < sources.length; i++) {
-      let network = yamlString(yamlField(sources[i], 'network'))
-      if (network !== null && validManifestNetwork(network)) {
+      let network = manifestMetadataString(yamlField(sources[i], 'network'), 256)
+      if (network !== null) {
         manifest.network = network
         return
       }
@@ -117,15 +123,6 @@ function readManifestNetwork(manifest: SubgraphDeploymentManifest, root: YAMLVal
   manifestWarning(manifest.id, 'network')
 }
 
-function validManifestNetwork(network: string): bool {
-  if (network.length > 256) return false
-  for (let i = 0; i < network.length; i++) {
-    let code = network.charCodeAt(i)
-    if (code <= 32 || code == 127) return false
-  }
-  return true
-}
-
 function readManifestDataSources(manifest: SubgraphDeploymentManifest, root: YAMLValue): void {
   let sources = yamlField(root, 'dataSources')
   if (sources === null || !sources.isArray() || sources.toArray().length == 0) {
@@ -152,8 +149,8 @@ function readManifestDataSources(manifest: SubgraphDeploymentManifest, root: YAM
       continue
     }
     let startBlockValue = yamlField(source, 'startBlock')
-    // A missing startBlock defaults to zero. An explicit null or malformed
-    // value is unknown, so we cannot reliably report a minimum.
+    // A missing startBlock defaults to zero. If any source is invalid, keep
+    // the manifest's zero default rather than reporting a partial minimum.
     let startBlock = startBlockValue === null ? BigInt.fromI32(0) : manifestStartBlock(startBlockValue)
     if (startBlock === null) {
       validStartBlocks = false
@@ -168,24 +165,44 @@ function readManifestDataSources(manifest: SubgraphDeploymentManifest, root: YAM
   else manifestWarning(manifest.id, 'dataSources.source.startBlock')
 }
 
+// Keep field extraction independent of YAML decoding so defaults also apply
+// to undecodable documents.
+export function parseSubgraphDeploymentManifest(
+  id: string,
+  content: string,
+  root: YAMLValue | null,
+): SubgraphDeploymentManifest {
+  let manifest = new SubgraphDeploymentManifest(id)
+  manifest.manifest = content
+  // Preserve the defaults exposed by the original parser, including when
+  // malformed or incomplete manifests prevent further field extraction.
+  manifest.startBlock = BigInt.fromI32(0)
+  manifest.poweredBySubstreams = false
+  if (root !== null && root.isObject()) {
+    readManifestSchema(manifest, root)
+    readManifestNetwork(manifest, root)
+    readManifestDataSources(manifest, root)
+  }
+  return manifest
+}
+
 export function handleSubgraphDeploymentManifest(content: Bytes): void {
-  let manifest = new SubgraphDeploymentManifest(dataSource.stringParam())
-  manifest.manifest = content.toString()
-  // Match the native parser's input limit; retain the raw manifest on failure.
+  let id = dataSource.stringParam()
+  let root: YAMLValue | null = null
+  // Match the native parser's input limit; retain the raw document and defaults
+  // when it cannot be decoded.
   if (content.length > 10000000) {
-    manifestWarning(manifest.id, 'manifest size')
-    manifest.save()
-    return
-  }
-  let parsed = yaml.try_fromBytes(content)
-  if (!parsed.isOk) {
-    manifestWarning(manifest.id, 'YAML')
-  } else if (!parsed.value.isObject()) {
-    manifestWarning(manifest.id, 'manifest root')
+    manifestWarning(id, 'manifest size')
   } else {
-    readManifestSchema(manifest, parsed.value)
-    readManifestNetwork(manifest, parsed.value)
-    readManifestDataSources(manifest, parsed.value)
+    let parsed = yaml.try_fromBytes(content)
+    if (!parsed.isOk) {
+      manifestWarning(id, 'YAML')
+    } else if (!parsed.value.isObject()) {
+      manifestWarning(id, 'manifest root')
+    } else {
+      root = parsed.value
+    }
   }
+  let manifest = parseSubgraphDeploymentManifest(id, content.toString(), root)
   manifest.save()
 }