Hello,
I’ve faced an issue while trying to get working with Corvid, wix-data and database collections, it behaves unpredictable and unstable.
I have a backend function that gets some data through HTTP from external API and then saves it to the Collection using wixData.bulkSave. I run this function via Job Scheduler.
My journey started here: a day ago during debugging of this job I noticed a bug/feature - after getting ~500 items from external api using fetch and trying to save one item to collection I’ve got Internal Server Error. After some time I figured out that wix-data save method fails but not due to wrong data, it fails because of the previous call to external api. Strange. So I decided that it is because of these 500 items from external api took a lot of memory. So, I started getting data from external API page by page with the smaller size, that’s where I met WDE0053: Internal wixData error: Unknown error.
Now when I’m getting data from external API page by page I predictable got WDE0053: Internal wixData error: Unknown error after 2-3 iterations of wixData.bulkSave. I’ve tried retries - didn’t help. This behavior doesn’t depend on data as if I rerun this job from iteration where it previously failed - it works several iterations and then the same failure. Looks like the state of the process which runs my function becomes invalid at some point and it’s not possible to save more data because if I rerun the job several iterations of saving will immediately work. I also tried to use wixData.save with a loop instead of wixData.bulkSave - same error.
I didn’t get any reply from support yet.
Any idea how to fix that? Thanks.
import wixData from 'wix-data';
import fetchShelterData from './Sheltermate/Sheltermate.js';
const timeout = ms => new Promise(res => setTimeout(res, ms))
async function fetchAndRefreshShelterData(api) {
let nextPage = 1;
while(nextPage) {
console.log("Fetching data from " + api.url);
let results = await fetchShelterData({
apiUrl: api.url,
page: nextPage,
pageSize: 50
})
nextPage = results.nextPage;
let animals = results.animals;
await saveAnimals(api, animals);
animals = null;
}
}
async function saveAnimals(api, animals) {
let dbAnimals = animals.map(animal => {
let dbItem = {
"_id": api.shelter + '-' + animal.id.toString(),
"shelter": api.shelter,
"title": animal.id + '-' + animal.name.replace(/\s/g, "-"),
....
}
return dbItem;
})
const max = 10;
let numberOfRetries = max;
while(numberOfRetries > 0) {
try {
let saveResult = await wixData.bulkSave("Pets", dbAnimals, {"suppressHooks": true});
numberOfRetries = 0;
console.log(saveResult);
} catch(e) {
numberOfRetries--;
if(numberOfRetries <=0) throw e;
console.log(e.name + ': ' + e.message);
console.log("Retrying save data..")
await timeout((max - numberOfRetries) * 1000);
}
}
}