Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0

### Added

- **`fetch()` with `outputFields` parameter** (#176)
- `ZVecCollection::fetch()` now accepts an optional output-fields array to select which scalar columns are returned: `fetch(['pk1', 'pk2'], ['name', 'score'])`.
- BC-compatible: the legacy variadic form `fetch('pk1', 'pk2')` (all fields) still works unchanged.
- Mirrors upstream zvec C API `zvec_collection_fetch` output-fields semantics (v0.5.0+, alibaba/zvec#358): unknown field names are silently ignored; vector fields are always included.
- FFI: `zvec_collection_fetch` extended with `output_fields` / `output_field_count` parameters (empty → all fields, matching upstream `std::nullopt` semantics).

### Fixed

- **Random rotation for INT8/INT4 quantization** (#177)
- `ZVecIndexParams::setQuantizerEnableRotate(bool)` (fluent) enables random rotation before INT8/INT4 quantization for HNSW, Flat, IVF, and Vamana indexes — reduces quantization error and improves recall on quantized indexes.
- Mirrors upstream zvec v0.6.0 `QuantizerParam(enable_rotate)` (C API: `zvec_index_params_set_quantizer_enable_rotate`).
Expand Down
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -236,7 +236,7 @@ $collection->upsertBatch(ZVecDoc ...$docs): array // Returns per-doc status
$collection->updateBatch(ZVecDoc ...$docs): array // Returns per-doc status array
$collection->delete(string ...$pks): void
$collection->deleteByFilter(string $filter): void
$collection->fetch(string ...$pks): ZVecDoc[]
$collection->fetch(string ...$pks): ZVecDoc[] // also fetch(array $pks, ?array $outputFields = null)

// Search
$collection->query(string|ZVecVectorQuery $fieldName, array $queryVector = [], int $topk = 10, ...): ZVecDoc[]
Expand Down
21 changes: 19 additions & 2 deletions ffi/zvec_ffi.cc
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

#include <array>
#include <cstring>
#include <optional>
#include <string>
#include <unordered_map>
#include <vector>
Expand Down Expand Up @@ -2351,7 +2352,9 @@ zvec_status_t zvec_collection_group_by_query_vector(zvec_collection_t coll, cons

// --- Fetch ---

zvec_status_t zvec_collection_fetch(zvec_collection_t coll, const char** pks, int count, zvec_query_result_t* result) {
zvec_status_t zvec_collection_fetch(zvec_collection_t coll, const char** pks, int count,
const char** output_fields, int output_field_count,
zvec_query_result_t* result) {
if (!coll) {
zvec_status_t st = {1, "null handle"};
SET_FFI_ERROR(st);
Expand All @@ -2363,7 +2366,21 @@ zvec_status_t zvec_collection_fetch(zvec_collection_t coll, const char** pks, in
for (int i = 0; i < count; i++) {
pk_vec.emplace_back(pks[i]);
}
auto res = c->Fetch(pk_vec);
std::optional<std::vector<std::string>> cpp_output_fields;
if (output_fields && output_field_count > 0) {
std::vector<std::string> fields;
fields.reserve(output_field_count);
for (int i = 0; i < output_field_count; i++) {
if (!output_fields[i]) {
zvec_status_t st = {1, "null output field"};
SET_FFI_ERROR(st);
return st;
}
fields.emplace_back(output_fields[i]);
}
cpp_output_fields = std::move(fields);
}
auto res = c->Fetch(pk_vec, cpp_output_fields);
if (!res.has_value()) {
result->docs = nullptr;
result->count = 0;
Expand Down
4 changes: 3 additions & 1 deletion ffi/zvec_ffi.h
Original file line number Diff line number Diff line change
Expand Up @@ -327,7 +327,9 @@ void zvec_group_by_vector_query_set_is_linear(zvec_group_by_vector_query_t q, in
void zvec_group_by_vector_query_set_using_refiner(zvec_group_by_vector_query_t q, int refiner);

// Fetch
zvec_status_t zvec_collection_fetch(zvec_collection_t coll, const char** pks, int count, zvec_query_result_t* result);
zvec_status_t zvec_collection_fetch(zvec_collection_t coll, const char** pks, int count,
const char** output_fields, int output_field_count,
zvec_query_result_t* result);

// Query
zvec_status_t zvec_collection_query(zvec_collection_t coll, const char* field_name,
Expand Down
4 changes: 3 additions & 1 deletion ffi/zvec_ffi_php.h
Original file line number Diff line number Diff line change
Expand Up @@ -251,7 +251,9 @@ zvec_status_t zvec_collection_upsert_batch(zvec_collection_t coll, zvec_doc_t* d
zvec_status_t zvec_collection_update_batch(zvec_collection_t coll, zvec_doc_t* docs, int count, zvec_batch_result_t* result);
void zvec_batch_result_free(zvec_batch_result_t* result);

zvec_status_t zvec_collection_fetch(zvec_collection_t coll, const char** pks, int count, zvec_query_result_t* result);
zvec_status_t zvec_collection_fetch(zvec_collection_t coll, const char** pks, int count,
const char** output_fields, int output_field_count,
zvec_query_result_t* result);

zvec_status_t zvec_collection_query(zvec_collection_t coll, const char* field_name,
const float* query_vector, uint32_t dim,
Expand Down
37 changes: 35 additions & 2 deletions src/ZVec.php
Original file line number Diff line number Diff line change
Expand Up @@ -739,24 +739,57 @@ public function deleteByFilter(string $filter): void
}

/**
* Fetch documents by PK. Accepts variadic PK strings (fetch('a', 'b')) or
* an array form with optional output fields (fetch(['a', 'b'], ['name'])).
*
* @return ZVecDoc[]
* @throws ZVecException On FFI error
*/
public function fetch(string ...$pks): array
public function fetch(array|string ...$args): array
{
$this->checkClosed();
if (empty($args)) {
throw new ZVecException('At least one PK is required');
}
if (is_array($args[0])) {
if (count($args) > 2) {
throw new ZVecException('Unexpected extra arguments: expected pks array and optional outputFields');
}
$pks = $args[0];
$outputFields = $args[1] ?? null;
} else {
$pks = $args;
$outputFields = null;
}
if (empty($pks)) {
throw new ZVecException('At least one PK is required');
}
foreach ($pks as $pk) {
if (!is_string($pk) || $pk === '') {
throw new ZVecException('PKs must be non-empty strings');
}
}
if ($outputFields !== null) {
if (!is_array($outputFields)) {
throw new ZVecException('outputFields must be an array of non-empty strings');
}
foreach ($outputFields as $field) {
if (!is_string($field) || $field === '') {
throw new ZVecException('outputFields must contain only non-empty strings');
}
}
}
$ffi = self::ffi();
[$arr, $count, $cStrings] = self::toCStringArray($ffi, $pks);
[$ofArr, $ofCount, $ofCStrings] = self::toCStringArray($ffi, $outputFields ?? []);

$result = $ffi->new('zvec_query_result_t');
try {
$status = $ffi->zvec_collection_fetch($this->handle, $arr, $count, FFI::addr($result));
$status = $ffi->zvec_collection_fetch($this->handle, $arr, $count, $ofArr, $ofCount, FFI::addr($result));
self::checkStatus($status);
} finally {
self::freeCStringArray($cStrings);
self::freeCStringArray($ofCStrings);
}

return self::parseQueryResult($result);
Expand Down
122 changes: 122 additions & 0 deletions tests/test_fetch_output_fields.phpt
Original file line number Diff line number Diff line change
@@ -0,0 +1,122 @@
--TEST--
Data operations: fetch with outputFields parameter
--SKIPIF--
<?php if (!extension_loaded('ffi')) die('skip FFI extension not available'); ?>
--FILE--
<?php
require_once __DIR__ . '/../src/ZVec.php';
set_error_handler(static fn(int $errno, string $message): bool => $errno === E_USER_DEPRECATED && str_contains($message, ' is deprecated, use createIndex()'));
ZVec::init(logType: ZVec::LOG_CONSOLE, logLevel: ZVec::LOG_WARN);

$path = __DIR__ . '/../test_dbs/fetch_output_fields_' . uniqid();
try {
$schema = new ZVecSchema('fetch_output_fields_test');
$schema->setMaxDocCountPerSegment(1000)
->addInt64('id', nullable: false, withInvertIndex: true)
->addString('name', nullable: true, withInvertIndex: true)
->addFloat('score', nullable: true)
->addDouble('rating', nullable: true)
->addVectorFp32('embedding', dimension: 4, metricType: ZVecSchema::METRIC_IP);

$c = ZVec::create($path, $schema);
$c->createHnswIndex('embedding', metricType: ZVecSchema::METRIC_IP, m: 16, efConstruction: 200);

for ($i = 1; $i <= 3; $i++) {
$doc = new ZVecDoc("doc$i");
$doc->setInt64('id', $i)
->setString('name', "User$i")
->setFloat('score', 80.0 + $i * 2)
->setDouble('rating', 3.0 + $i * 0.5)
->setVectorFp32('embedding', [1.0 * $i, 0.0, 0.0, 0.0]);
$c->insert($doc);
}
echo "Inserted 3 documents\n";

// outputFields subset: only requested scalar fields returned (vector always
// included because include_vector is not exposed and defaults to true)
$fetched = $c->fetch(['doc1', 'doc2'], ['name']);
assert(count($fetched) === 2, 'Should fetch 2 documents');
foreach ($fetched as $d) {
assert($d->getString('name') === 'User' . substr($d->getPk(), 3), 'name should be present');
assert($d->getInt64('id') === null, 'id should be excluded');
assert($d->getFloat('score') === null, 'score should be excluded');
assert($d->getDouble('rating') === null, 'rating should be excluded');
assert($d->getVectorFp32('embedding') !== null, 'vector should always be present (include_vector)');
}
echo "outputFields subset OK\n";

// fetch without outputFields still returns all fields (BC)
$fetched = $c->fetch('doc1');
assert(count($fetched) === 1, 'Should fetch 1 document');
assert($fetched[0]->getInt64('id') === 1, 'id should be present');
assert($fetched[0]->getString('name') === 'User1', 'name should be present');
assert($fetched[0]->getFloat('score') === 82.0, 'score should be present');
assert($fetched[0]->getDouble('rating') === 3.5, 'rating should be present');
assert($fetched[0]->getVectorFp32('embedding') !== null, 'vector should be present');
echo "Fetch all fields BC OK\n";

// array form fetch(['a', 'b'], ['field'])
$fetched = $c->fetch(['doc1', 'doc3'], ['score']);
assert(count($fetched) === 2, 'Should fetch 2 documents');
foreach ($fetched as $d) {
assert($d->getFloat('score') === 80.0 + substr($d->getPk(), 3) * 2, 'score should be present');
assert($d->getString('name') === null, 'name should be excluded');
}
echo "Array form with outputFields OK\n";

// unknown output field names are silently ignored (only existing fields returned)
$fetched = $c->fetch(['doc1'], ['nonexistent_field', 'also_missing']);
assert(count($fetched) === 1, 'Should fetch 1 document');
assert($fetched[0]->getString('name') === null, 'unrequested field should be null');
assert($fetched[0]->getInt64('id') === null, 'unrequested field should be null');
echo "Unknown output fields ignored OK\n";

// invalid outputFields (non-string entries) -> ZVecException
try {
$c->fetch(['doc1'], ['name', 123]);
assert(false, 'Should throw ZVecException for non-string output field');
} catch (ZVecException $e) {
echo "Invalid outputFields type throws OK\n";
}

// invalid outputFields (empty string entry) -> ZVecException
try {
$c->fetch(['doc1'], ['']);
assert(false, 'Should throw ZVecException for empty output field');
} catch (ZVecException $e) {
echo "Invalid outputFields empty string throws OK\n";
}

// invalid PK entry -> ZVecException
try {
$c->fetch(['doc1', 42]);
assert(false, 'Should throw ZVecException for non-string PK');
} catch (ZVecException $e) {
echo "Invalid PK throws OK\n";
}

// extra arguments after outputFields -> ZVecException
try {
$c->fetch(['doc1'], ['name'], 'extra');
assert(false, 'Should throw ZVecException for extra arguments');
} catch (ZVecException $e) {
echo "Extra arguments throw OK\n";
}

$c->close();
echo "PASS: fetch with outputFields works\n";
} finally {
exec("rm -rf " . escapeshellarg($path));
}
?>
--EXPECT--
Inserted 3 documents
outputFields subset OK
Fetch all fields BC OK
Array form with outputFields OK
Unknown output fields ignored OK
Invalid outputFields type throws OK
Invalid outputFields empty string throws OK
Invalid PK throws OK
Extra arguments throw OK
PASS: fetch with outputFields works
Loading