我有一个非常大的文件（55 GB 的 json 数据）。我正在使用ifstream读取和另一个ofstream写入另一个文件。该程序可以正常运行一段时间。然后由于大量内存使用而崩溃。
我尝试使用ignore和clear清除输入缓冲区。并尝试使用flush清除输出缓冲区。
该文件也非常大，所以，我希望它很快。
ps我在半睡半醒的时候写了json解析器。所以请原谅我糟糕的解析器代码。也许那里存在内存泄漏。任何帮助将不胜感激。
小例子
int main()
{
std::ifstream file("aggressive_dedup.json", std::ifstream::in);
std::ofstream outFile("processed.json", std::ofstream::out);
std::string str;
int a;
long long count = 0;
while (std::getline(file, str))
{
JsonParserStateMachine jsonParserStateMachine;
for(char &c : str) jsonParserStateMachine.changeState(c);
//std::cout<<jsonParserStateMachine.getReview();
//This lines just gives a string to the output which is around may be 1000 characters
outFile << jsonParserStateMachine.getReview();
if(++count % 1000 == 0) {
std::cout<<count<<" Processed\n";
outFile.flush();
return 0;
}
}
outFile.close();
return 0;
}
对于那些愿意看到整个代码的人
#include <fstream>
#include <string>
#include <iostream>
enum state {
q0, q1, q2, q3, q4, q5, q6, h
};
class KeyValueStore{
std::string *keys;
std::string *values;
int currentKeyPosition;
int currentValuePosition;
int maxLength;
public:
KeyValueStore(const int length) : maxLength(length),currentKeyPosition(0),currentValuePosition(0)
{
this->keys = new std::string[length];
this->values = new std::string[length];
for(int i=0;i<length;i++)
{
this->keys[i] = "";
this->values[i] = "";
}
}
void updateKeyPosition()
{
this->currentKeyPosition = this->currentKeyPosition++%9;
}
void updateValuePosition()
{
this->currentValuePosition = this->currentValuePosition++%9;
}
void putKey(char c)
{
this->keys[currentKeyPosition] += c;
}
void putValue(char c)
{
this->values[currentValuePosition] += c;
}
std::string getValue(std::string key)
{
for(int i=0;i<this->maxLength;i++)
{
if(this->keys[i] == key) return this->values[i];
}
return "";
}
void print()
{
std::cout<<"Keys"<<"\t"<<"Values"<<std::endl;
for(int i=0;i<maxLength;i++)
{
std::cout<<this->keys[i] <<'\t'<<this->values[i]<<std::endl;
}
}
std::string getReview()
{
return std::string("{\"" + this->getValue("reviewText") + "\":\"" + this->getValue("overall") + "\"}");
}
};
class JsonParserStateMachine{
state currentState;
KeyValueStore keyValueStore;
bool inNum;
bool inArray;
public:
JsonParserStateMachine(): keyValueStore(9), currentState(state::q0), inNum(false),inArray(false){}
state getState()
{
return this->currentState;
}
void print()
{
keyValueStore.print();
}
std::string getReview()
{
return keyValueStore.getReview();
}
state changeState(char c)
{
switch(currentState)
{
case state::q0:
if(c == ' ') break;
else if(c == '{') this->currentState = state::q1;
else this->currentState = state::h;
break;
case state::q1:
if(c == ' ') break;
else if(c == '\"') this->currentState = state::q2;
else this->currentState = state::h;
break;
case state::q2:
if(c == '\"'){
this->currentState = state::q3;
this->keyValueStore.updateKeyPosition();
break;
}
else{
this->keyValueStore.putKey(c);
break;
}
case state::q3:
if(c == ':') this->currentState = state::q4;
else if(c == ' ') {
}
else {
//std::cout<<"From Q3"<<std::endl;
this->currentState = state::h;
}break;
case state::q4:
if(c == '\"' || c == '[') {
this->currentState = state::q5;
inArray = c == '[' ? true: false;
}else if(c == ' ') break;
else {
//std::cout<<"Got Here"<<std::endl;
inNum = true;
this->currentState = state::q5;
this->keyValueStore.putValue(c);
}
break;
case state::q5:
if(c == '\"' || c == ']'){
this->currentState = state::q6;
this->keyValueStore.updateValuePosition();
inArray = c == ']'? false: true;
break;
}else if(inNum && c == ',' ){
this->currentState = state::q1;
this->keyValueStore.updateValuePosition();
inNum = false;
}
else{
this->keyValueStore.putValue(c);
break;
}
case state::q6:
if(c == ','){
this->currentState = state::q1;
break;
}else if(c == ' '){
break;
}else{
//std::cout<<"From Q6"<<std::endl;
this->currentState = state::h;
}
}
return this->currentState;
}
};
class Review{
std::string reviewText;
int overall;
std::string summary;
public:
void pusReviewText(std::string reviewText)
{
this->reviewText = reviewText;
}
void putOverall(int overall)
{
this->overall = overall;
}
void putSummary(std::string summary)
{
this->summary = summary;
}
std::string getReviewText()
{
return this->reviewText;
}
int getOverall()
{
return this->overall;
}
std::string getSummary()
{
return this->summary;
}
};
int main()
{
std::ifstream file("aggressive_dedup.json", std::ifstream::in);
std::ofstream outFile("processed.json", std::ofstream::out);
std::string str;
int a;
long long count = 0;
while (std::getline(file, str))
{
JsonParserStateMachine jsonParserStateMachine;
for(char &c : str) jsonParserStateMachine.changeState(c);
//std::cout<<jsonParserStateMachine.getReview();
outFile << jsonParserStateMachine.getReview();
if(++count % 1000 == 0) {
std::cout<<count<<" Processed\n";
outFile.flush();
return 0;
}
}
outFile.close();
return 0;
}
问题来自您的KeyValueStore课程：
KeyValueStore(const int length) : maxLength(length),currentKeyPosition(0),currentValuePosition(0)
{
this->keys = new std::string[length];
this->values = new std::string[length];
...
没有任何东西会删除这些数组。在析构函数中删除它们是一个简单的修复：
~KeyValueStore() {
delete[] this->keys;
delete[] this->values;
}
然而！你真的应该考虑使用std::vector<std::string>。或者甚至更好，围绕一个std::unordered_map<std::string, std::string> instead.
本文收集自互联网，转载请注明来源。
如有侵权，请联系[email protected] 删除。
我来说两句