mirror of https://github.com/ghostfolio/ghostfolio
You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
419 lines
13 KiB
419 lines
13 KiB
#include "lmdb-js.h"
|
|
#include <cstdio>
|
|
|
|
using namespace Napi;
|
|
|
|
void setFlagFromValue(int *flags, int flag, const char *name, bool defaultValue, Object options);
|
|
|
|
DbiWrap::DbiWrap(const Napi::CallbackInfo& info) : ObjectWrap<DbiWrap>(info) {
|
|
this->dbi = 0;
|
|
this->keyType = LmdbKeyType::DefaultKey;
|
|
this->compression = nullptr;
|
|
this->isOpen = false;
|
|
this->getFast = false;
|
|
this->ew = nullptr;
|
|
EnvWrap *ew;
|
|
napi_unwrap(info.Env(), info[0], (void**) &ew);
|
|
this->env = ew->env;
|
|
this->ew = ew;
|
|
int flags = info[1].As<Number>();
|
|
char* nameBytes;
|
|
std::string name;
|
|
if (info[2].IsString()) {
|
|
name = info[2].As<String>().Utf8Value();
|
|
nameBytes = (char*) name.c_str();
|
|
} else
|
|
nameBytes = nullptr;
|
|
LmdbKeyType keyType = (LmdbKeyType) info[3].As<Number>().Int32Value();
|
|
Compression* compression;
|
|
if (info[4].IsObject())
|
|
napi_unwrap(info.Env(), info[4], (void**) &compression);
|
|
else
|
|
compression = nullptr;
|
|
int rc = this->open(flags, nameBytes, flags & HAS_VERSIONS,
|
|
keyType, compression);
|
|
//if (nameBytes)
|
|
//delete nameBytes;
|
|
if (rc) {
|
|
if (rc == MDB_NOTFOUND)
|
|
this->dbi = (MDB_dbi) 0xffffffff;
|
|
else {
|
|
//delete this;
|
|
throwLmdbError(info.Env(), rc);
|
|
return;
|
|
}
|
|
}
|
|
info.This().As<Object>().Set("dbi", Number::New(info.Env(), this->dbi));
|
|
info.This().As<Object>().Set("address", Number::New(info.Env(), (size_t) this));
|
|
}
|
|
|
|
|
|
DbiWrap::~DbiWrap() {
|
|
// Imagine the following JS:
|
|
// ------------------------
|
|
// var dbi1 = env.openDbi({ name: "hello" });
|
|
// var dbi2 = env.openDbi({ name: "hello" });
|
|
// dbi1.close();
|
|
// txn.putString(dbi2, "world");
|
|
// -----
|
|
// The above DbiWrap objects would both wrap the same MDB_dbi, and if closing the first one called mdb_dbi_close,
|
|
// that'd also render the second DbiWrap instance unusable.
|
|
//
|
|
// For this reason, we will never call mdb_dbi_close
|
|
// NOTE: according to LMDB authors, it is perfectly fine if mdb_dbi_close is never called on an MDB_dbi
|
|
}
|
|
|
|
|
|
int DbiWrap::open(int flags, char* name, bool hasVersions, LmdbKeyType keyType, Compression* compression) {
|
|
MDB_txn* txn = ew->getReadTxn();
|
|
this->hasVersions = hasVersions;
|
|
this->compression = compression;
|
|
this->keyType = keyType;
|
|
#ifndef MDB_RPAGE_CACHE
|
|
flags &= ~0x100; // remove use versions flag for older lmdb
|
|
#endif
|
|
this->flags = flags;
|
|
int rc = txn ? mdb_dbi_open(txn, name, flags, &this->dbi) : EINVAL;
|
|
if (rc)
|
|
return rc;
|
|
this->isOpen = true;
|
|
if (keyType == LmdbKeyType::DefaultKey && name) { // use the fast compare, but can't do it if we have db table/names mixed in
|
|
mdb_set_compare(txn, dbi, compareFast);
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
Value DbiWrap::close(const Napi::CallbackInfo& info) {
|
|
if (this->isOpen) {
|
|
mdb_dbi_close(this->env, this->dbi);
|
|
this->isOpen = false;
|
|
this->ew = nullptr;
|
|
}
|
|
else {
|
|
return throwError(info.Env(), "The Dbi is not open, you can't close it.");
|
|
}
|
|
return info.Env().Undefined();
|
|
}
|
|
|
|
Value DbiWrap::drop(const Napi::CallbackInfo& info) {
|
|
int del = 1;
|
|
int rc;
|
|
if (!this->isOpen) {
|
|
return throwError(info.Env(), "The Dbi is not open, you can't drop it.");
|
|
}
|
|
|
|
// Check if the database should be deleted
|
|
if (info.Length() == 1 && info[0].IsObject()) {
|
|
Napi::Object options = info[0].As<Object>();
|
|
|
|
// Just free pages
|
|
Napi::Value opt = options.Get("justFreePages");
|
|
del = opt.IsBoolean() ? !opt.As<Boolean>().Value() : 1;
|
|
}
|
|
|
|
// Drop database
|
|
rc = mdb_drop(ew->writeTxn->txn, dbi, del);
|
|
if (rc != 0) {
|
|
return throwLmdbError(info.Env(), rc);
|
|
}
|
|
|
|
// Only close database if del == 1
|
|
if (del == 1) {
|
|
isOpen = false;
|
|
ew = nullptr;
|
|
}
|
|
return info.Env().Undefined();
|
|
}
|
|
|
|
Value DbiWrap::stat(const Napi::CallbackInfo& info) {
|
|
MDB_stat stat;
|
|
mdb_stat(this->ew->getReadTxn(), dbi, &stat);
|
|
Object stats = Object::New(info.Env());
|
|
stats.Set("pageSize", Number::New(info.Env(), stat.ms_psize));
|
|
stats.Set("treeDepth", Number::New(info.Env(), stat.ms_depth));
|
|
stats.Set("treeBranchPageCount", Number::New(info.Env(), stat.ms_branch_pages));
|
|
stats.Set("treeLeafPageCount", Number::New(info.Env(), stat.ms_leaf_pages));
|
|
stats.Set("entryCount", Number::New(info.Env(), stat.ms_entries));
|
|
stats.Set("overflowPages", Number::New(info.Env(), stat.ms_overflow_pages));
|
|
return stats;
|
|
}
|
|
|
|
int32_t DbiWrap::doGetByBinary(uint32_t keySize, uint32_t ifNotTxnId, int64_t txnWrapAddress) {
|
|
char* keyBuffer = ew->keyBuffer;
|
|
MDB_txn* txn = ew->getReadTxn(txnWrapAddress);
|
|
MDB_val key, data;
|
|
key.mv_size = keySize;
|
|
key.mv_data = (void*) keyBuffer;
|
|
uint32_t* currentTxnId = (uint32_t*) (keyBuffer + 32);
|
|
#ifdef MDB_RPAGE_CACHE
|
|
int result = mdb_get_with_txn(txn, dbi, &key, &data, (mdb_size_t*) currentTxnId);
|
|
#else
|
|
int result = mdb_get(txn, dbi, &key, &data);
|
|
#endif
|
|
if (result) {
|
|
if (result > 0)
|
|
return -result;
|
|
return result;
|
|
}
|
|
#ifdef MDB_RPAGE_CACHE
|
|
if (ifNotTxnId && ifNotTxnId == *currentTxnId)
|
|
return -30004;
|
|
#endif
|
|
result = getVersionAndUncompress(data, this);
|
|
bool fits = true;
|
|
if (result) {
|
|
fits = valToBinaryFast(data, this); // it fits in the global/compression-target buffer
|
|
}
|
|
#if ENABLE_V8_API
|
|
// TODO: We may want to enable this for Bun as well, since it probably doesn't have the same
|
|
// shared pointer problems that V8 does
|
|
if (fits || result == 2 || data.mv_size < SHARED_BUFFER_THRESHOLD) {// result = 2 if it was decompressed
|
|
#endif
|
|
if (data.mv_size < 0x80000000)
|
|
return data.mv_size;
|
|
*((uint32_t*)keyBuffer) = data.mv_size;
|
|
return -30000;
|
|
#if ENABLE_V8_API
|
|
} else {
|
|
return EnvWrap::toSharedBuffer(ew->env, (uint32_t*) ew->keyBuffer, data);
|
|
}
|
|
#endif
|
|
}
|
|
|
|
NAPI_FUNCTION(directWrite) {
|
|
ARGS(5)
|
|
GET_INT64_ARG(0);
|
|
DbiWrap* dw = (DbiWrap*) i64;
|
|
uint32_t keySize;
|
|
GET_UINT32_ARG(keySize, 1);
|
|
uint32_t offset;
|
|
GET_UINT32_ARG(offset, 2);
|
|
uint32_t dataSize;
|
|
GET_UINT32_ARG(dataSize, 3);
|
|
int64_t txnAddress = 0;
|
|
napi_status status = napi_get_value_int64(env, args[4], &txnAddress);
|
|
if (dw->hasVersions) offset += 8;
|
|
EnvWrap* ew = dw->ew;
|
|
char* keyBuffer = ew->keyBuffer;
|
|
MDB_txn* txn = ew->getReadTxn(txnAddress);
|
|
MDB_val key, data;
|
|
key.mv_size = keySize;
|
|
key.mv_data = (void*) keyBuffer;
|
|
data.mv_size = dataSize;
|
|
data.mv_data = (void*) (keyBuffer + (((keySize >> 3) + 1) << 3));
|
|
#ifdef MDB_RPAGE_CACHE
|
|
int result = mdb_direct_write(txn, dw->dbi, &key, offset, &data);
|
|
#else
|
|
int result = -1;
|
|
#endif
|
|
RETURN_INT32(result);
|
|
}
|
|
|
|
NAPI_FUNCTION(getByBinary) {
|
|
ARGS(4)
|
|
GET_INT64_ARG(0);
|
|
DbiWrap* dw = (DbiWrap*) i64;
|
|
uint32_t keySize;
|
|
GET_UINT32_ARG(keySize, 1);
|
|
uint32_t ifNotTxnId;
|
|
GET_UINT32_ARG(ifNotTxnId, 2);
|
|
int64_t txnAddress = 0;
|
|
napi_status status = napi_get_value_int64(env, args[3], &txnAddress);
|
|
RETURN_INT32(dw->doGetByBinary(keySize, ifNotTxnId, txnAddress));
|
|
}
|
|
|
|
uint32_t getByBinaryFFI(double dwPointer, uint32_t keySize, uint32_t ifNotTxnId, uint64_t txnAddress) {
|
|
DbiWrap* dw = (DbiWrap*) (size_t) dwPointer;
|
|
return dw->doGetByBinary(keySize, ifNotTxnId, txnAddress);
|
|
}
|
|
|
|
napi_finalize noopDbi = [](napi_env, void *, void *) {
|
|
// Data belongs to LMDB, we shouldn't free it here
|
|
};
|
|
NAPI_FUNCTION(getSharedByBinary) {
|
|
ARGS(2)
|
|
GET_INT64_ARG(0);
|
|
DbiWrap* dw = (DbiWrap*) i64;
|
|
uint32_t keySize;
|
|
GET_UINT32_ARG(keySize, 1);
|
|
MDB_val key;
|
|
MDB_val data;
|
|
key.mv_size = keySize;
|
|
key.mv_data = (void*) dw->ew->keyBuffer;
|
|
MDB_txn* txn = dw->ew->getReadTxn();
|
|
int rc = mdb_get(txn, dw->dbi, &key, &data);
|
|
if (rc) {
|
|
if (rc == MDB_NOTFOUND) {
|
|
RETURN_UNDEFINED;
|
|
} else
|
|
return throwLmdbError(env, rc);
|
|
}
|
|
rc = getVersionAndUncompress(data, dw);
|
|
napi_create_external_buffer(env, data.mv_size,
|
|
(char*) data.mv_data, noopDbi, nullptr, &returnValue);
|
|
return returnValue;
|
|
}
|
|
NAPI_FUNCTION(getStringByBinary) {
|
|
ARGS(3)
|
|
GET_INT64_ARG(0);
|
|
DbiWrap* dw = (DbiWrap*) i64;
|
|
uint32_t keySize;
|
|
GET_UINT32_ARG(keySize, 1);
|
|
int64_t txnAddress = 0;
|
|
napi_status status = napi_get_value_int64(env, args[2], &txnAddress);
|
|
MDB_val key;
|
|
MDB_val data;
|
|
key.mv_size = keySize;
|
|
key.mv_data = (void*) dw->ew->keyBuffer;
|
|
MDB_txn* txn = dw->ew->getReadTxn(txnAddress);
|
|
int rc = mdb_get(txn, dw->dbi, &key, &data);
|
|
if (rc) {
|
|
if (rc == MDB_NOTFOUND) {
|
|
RETURN_UNDEFINED;
|
|
} else
|
|
return throwLmdbError(env, rc);
|
|
}
|
|
rc = getVersionAndUncompress(data, dw);
|
|
if (rc)
|
|
napi_create_string_utf8(env, (char*) data.mv_data, data.mv_size, &returnValue);
|
|
else
|
|
napi_create_int32(env, data.mv_size, &returnValue);
|
|
return returnValue;
|
|
}
|
|
|
|
int DbiWrap::prefetch(uint32_t* keys) {
|
|
MDB_txn* txn = ExtendedEnv::getPrefetchReadTxn(ew->env);
|
|
MDB_val key;
|
|
MDB_val data;
|
|
unsigned int flags;
|
|
mdb_dbi_flags(txn, dbi, &flags);
|
|
bool findAllValues = flags & MDB_DUPSORT;
|
|
int effected = 0;
|
|
bool findDataValue = false;
|
|
MDB_cursor *cursor;
|
|
int rc = mdb_cursor_open(txn, dbi, &cursor);
|
|
if (rc) {
|
|
ExtendedEnv::donePrefetchReadTxn(txn);
|
|
return rc;
|
|
}
|
|
|
|
while((key.mv_size = *keys++) > 0) {
|
|
if (key.mv_size == 0xffffffff) {
|
|
// it is a pointer to a new buffer
|
|
keys = (uint32_t*) (size_t) *((double*) keys); // read as a double pointer
|
|
key.mv_size = *keys++;
|
|
if (key.mv_size == 0)
|
|
break;
|
|
}
|
|
if (key.mv_size & 0x80000000) {
|
|
// indicator of using a data value combination (with dupSort), to be followed by the corresponding key
|
|
data.mv_size = key.mv_size & 0x7fffffff;
|
|
data.mv_data = (void *) keys;
|
|
keys += (data.mv_size + 12) >> 2;
|
|
findDataValue = true;
|
|
findAllValues = false;
|
|
continue;
|
|
}
|
|
// else standard key
|
|
key.mv_data = (void *) keys;
|
|
keys += (key.mv_size + 12) >> 2;
|
|
int rc = mdb_cursor_get(cursor, &key, &data, findDataValue ? MDB_GET_BOTH : MDB_SET_KEY);
|
|
findDataValue = false;
|
|
while (!rc) {
|
|
// access one byte from each of the pages to ensure they are in the OS cache,
|
|
// potentially triggering the hard page fault in this thread
|
|
int pages = (data.mv_size + 0xfff) >> 12;
|
|
// TODO: Adjust this for the page headers, I believe that makes the first page slightly less 4KB.
|
|
for (int i = 0; i < pages; i++) {
|
|
effected += *(((uint8_t*)data.mv_data) + (i << 12));
|
|
}
|
|
if (findAllValues) // in dupsort databases, access the rest of the values
|
|
rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT_DUP);
|
|
else
|
|
rc = 1; // done
|
|
}
|
|
}
|
|
mdb_cursor_close(cursor);
|
|
ExtendedEnv::donePrefetchReadTxn(txn);
|
|
return effected;
|
|
}
|
|
|
|
class PrefetchWorker : public AsyncWorker {
|
|
public:
|
|
PrefetchWorker(DbiWrap* dw, uint32_t* keys, const Function& callback)
|
|
: AsyncWorker(callback), dw(dw), keys(keys) {}
|
|
|
|
void Execute() {
|
|
dw->prefetch(keys);
|
|
}
|
|
|
|
void OnOK() {
|
|
napi_value result; // we use direct napi call here because node-addon-api interface with throw a fatal error if a worker thread is terminating
|
|
napi_call_function(Env(), Env().Undefined(), Callback().Value(), 0, {}, &result);
|
|
}
|
|
void OnError(const Error& e) {
|
|
napi_value result; // we use direct napi call here because node-addon-api interface with throw a fatal error if a worker thread is terminating
|
|
napi_value arg = e.Value();
|
|
napi_call_function(Env(), Env().Undefined(), Callback().Value(), 1, &arg, &result);
|
|
}
|
|
|
|
private:
|
|
DbiWrap* dw;
|
|
uint32_t* keys;
|
|
};
|
|
|
|
NAPI_FUNCTION(prefetchNapi) {
|
|
ARGS(3)
|
|
GET_INT64_ARG(0);
|
|
DbiWrap* dw = (DbiWrap*) i64;
|
|
napi_get_value_int64(env, args[1], &i64);
|
|
uint32_t* keys = (uint32_t*) i64;
|
|
PrefetchWorker* worker = new PrefetchWorker(dw, keys, Function(env, args[2]));
|
|
worker->Queue();
|
|
RETURN_UNDEFINED;
|
|
}
|
|
|
|
void DbiWrap::setupExports(Napi::Env env, Object exports) {
|
|
Function DbiClass = DefineClass(env, "Dbi", {
|
|
// DbiWrap: Prepare constructor template
|
|
// DbiWrap: Add functions to the prototype
|
|
DbiWrap::InstanceMethod("close", &DbiWrap::close),
|
|
DbiWrap::InstanceMethod("drop", &DbiWrap::drop),
|
|
DbiWrap::InstanceMethod("stat", &DbiWrap::stat),
|
|
});
|
|
exports.Set("Dbi", DbiClass);
|
|
EXPORT_NAPI_FUNCTION("directWrite", directWrite);
|
|
EXPORT_NAPI_FUNCTION("getByBinary", getByBinary);
|
|
EXPORT_NAPI_FUNCTION("prefetch", prefetchNapi);
|
|
EXPORT_NAPI_FUNCTION("getStringByBinary", getStringByBinary);
|
|
EXPORT_NAPI_FUNCTION("getSharedByBinary", getSharedByBinary);
|
|
EXPORT_FUNCTION_ADDRESS("getByBinaryPtr", getByBinaryFFI);
|
|
// TODO: wrap mdb_stat too
|
|
}
|
|
|
|
|
|
|
|
|
|
// This file contains code from the node-lmdb project
|
|
// Copyright (c) 2013-2017 Timur Kristóf
|
|
// Copyright (c) 2021 Kristopher Tate
|
|
// Licensed to you under the terms of the MIT license
|
|
//
|
|
// Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
// of this software and associated documentation files (the "Software"), to deal
|
|
// in the Software without restriction, including without limitation the rights
|
|
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
// copies of the Software, and to permit persons to whom the Software is
|
|
// furnished to do so, subject to the following conditions:
|
|
|
|
// The above copyright notice and this permission notice shall be included in
|
|
// all copies or substantial portions of the Software.
|
|
|
|
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
|
// THE SOFTWARE.
|
|
|
|
|