Skip to content

Commit

Permalink
updatelistmanager, use database file to store temp links
Browse files Browse the repository at this point in the history 
fix #145
  • Loading branch information
@riderkick
riderkick committed Dec 29, 2015
1 parent 74ce180 commit 148c3a6
Showing 1 changed file with 51 additions and 90 deletions.
141 changes: 51 additions & 90 deletions baseunits/uUpdateThread.pas
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,8 @@ TUpdateMangaManagerThread = class(TFMDThread)
CS_AddInfoToData, CS_AddNamesAndLinks: TCriticalSection;
isFinishSearchingForNewManga, isDownloadFromServer, isDoneUpdateNecessary: Boolean;
mainDataProcess: TDBDataProcess;
names, links, websites: TStringList;
tempDataProcess: TDBDataProcess;
websites: TStringList;
website, twebsite: String;
ModuleId: Integer;
workPtr, directoryCount,
Expand Down Expand Up @@ -109,6 +110,7 @@ destructor TUpdateMangaThread.Destroy;
procedure TUpdateMangaThread.Execute;
var
names, links: TStringList;
name, link: String;
i: Integer;
begin
try
Expand Down Expand Up @@ -194,13 +196,14 @@ procedure TUpdateMangaThread.Execute;
if links.Count > 0 then
begin
if manager.SortedList then
if manager.mainDataProcess.LinkExist(links.Strings[0]) then
if manager.mainDataProcess.LinkExist(links[0]) then
manager.isFinishSearchingForNewManga := True;

manager.CS_AddNamesAndLinks.Acquire;
try
manager.links.AddStrings(links);
manager.names.AddStrings(names);
for i := 0 to links.Count - 1 do
manager.tempDataProcess.AddData(names[i],links[i],'','','','','',0,0);
manager.tempDataProcess.Commit;
finally
manager.CS_AddNamesAndLinks.Release;
end;
Expand All @@ -213,14 +216,15 @@ procedure TUpdateMangaThread.Execute;

CS_INFO:
begin
Info.mangaInfo.title := manager.names[workPtr];
Info.GetInfoFromURL(manager.website, manager.links[workPtr],
OptionConnectionMaxRetry);
name := manager.tempDataProcess.Value[workPtr,0];
link := manager.tempDataProcess.Value[workPtr,1];
Info.mangaInfo.title := name;
Info.GetInfoFromURL(manager.website, link,OptionConnectionMaxRetry);
if not Terminated then
begin
manager.CS_AddInfoToData.Acquire;
try
Info.AddInfoToData(manager.names[workPtr], manager.links[workPtr],
Info.AddInfoToData(name, link,
manager.mainDataProcess);
manager.CheckCommit(32);
finally
Expand All @@ -238,8 +242,8 @@ procedure TUpdateMangaThread.Execute;
if checkStyle = CS_INFO then
begin
E.Message := E.Message +
' Title : ' + manager.names[workPtr] + LineEnding +
' URL : ' + manager.links[workPtr] + LineEnding;
' Title : ' + name + LineEnding +
' URL : ' + link + LineEnding;
end;
MainForm.ExceptionHandler(Self, E);
end;
Expand Down Expand Up @@ -329,10 +333,9 @@ constructor TUpdateMangaManagerThread.Create;
FreeOnTerminate := True;

websites := TStringList.Create;
names := TStringList.Create;
links := TStringList.Create;

mainDataProcess := TDBDataProcess.Create;
tempDataProcess := TDBDataProcess.Create;

threads := TFPList.Create;
SortedList := False;
Expand All @@ -342,12 +345,13 @@ constructor TUpdateMangaManagerThread.Create;

destructor TUpdateMangaManagerThread.Destroy;
begin
websites.Free;
mainDataProcess.Close;
DeleteDBDataProcess(twebsite);
websites.Free;
names.Free;
links.Free;
tempDataProcess.Close;
DeleteDBDataProcess('__tempupdatelist');
mainDataProcess.Free;
tempDataProcess.Free;
threads.Free;
MainForm.isUpdating := False;
CS_AddInfoToData.Free;
Expand Down Expand Up @@ -402,7 +406,7 @@ procedure TUpdateMangaManagerThread.RefreshList;

procedure TUpdateMangaManagerThread.DlgReport;
begin
MessageDlg('', Format(RS_DlgHasNewManga, [website, links.Count]),
MessageDlg('', Format(RS_DlgHasNewManga, [website, tempDataProcess.RecordCount]),
mtInformation, [mbYes], 0);
end;

Expand Down Expand Up @@ -492,7 +496,7 @@ procedure TUpdateMangaManagerThread.GetInfo(const limit: Integer;
CS_DIRECTORY_PAGE_2:
s := s + ' | ' + RS_LookingForNewTitleFromAnotherDirectory + '...';
CS_INFO:
s := Format('%s | %s "%s"', [s, RS_GettingInfo, names[workPtr - 1]]);
s := Format('%s | %s "%s"', [s, RS_GettingInfo, tempDataProcess.Value[workPtr-1,0]]);
end;
FStatus := s;
MainForm.ulWorkPtr := workPtr + 1;
Expand Down Expand Up @@ -531,7 +535,6 @@ procedure TUpdateMangaManagerThread.DoTerminate;
procedure TUpdateMangaManagerThread.Execute;
var
c, j, k: Integer;
del: Boolean;
begin
if websites.Count = 0 then
Exit;
Expand All @@ -551,6 +554,7 @@ procedure TUpdateMangaManagerThread.Execute;
end;
end
else
tempDataProcess.CreateDatabase('__tempupdatelist');
while websitePtr < websites.Count do
begin
website := websites.Strings[websitePtr];
Expand Down Expand Up @@ -581,15 +585,15 @@ procedure TUpdateMangaManagerThread.Execute;
mainDataProcess.InitLocateLink;
mainDataProcess.CloseTable;

//get directory page count
// get directory page count
INIAdvanced.Reload;
directoryCount := 0;
directoryCount2 := 0;
workPtr := 0;
GetInfo(1, CS_DIRECTORY_COUNT);
if Terminated then Break;

//get names and links
// get names and links
INIAdvanced.Reload;
workPtr := 0;
isFinishSearchingForNewManga := False;
Expand Down Expand Up @@ -625,92 +629,52 @@ procedure TUpdateMangaManagerThread.Execute;
[websitePtr, websites.Count, website]) + ' | ' + RS_IndexingNewTitle + '...';
Synchronize(MainThreadShowGetting);

// remove duplicate
if links.Count > 0 then
begin
c := 0;
j := 0;
MainForm.ulTotalPtr := links.Count;
MainForm.ulWorkPtr := j;
FStatus := RS_UpdatingList + Format(' [%d/%d] %s',
[websitePtr, websites.Count, website]) + ' | ' + RS_RemovingDuplicateFromNewTitle + '...';
Synchronize(MainThreadShowGetting);
while j < (links.Count - 1) do
begin
if Terminated then Break;
Inc(c);
if c > 499 then
begin
c := 0;
MainForm.ulTotalPtr := links.Count;
MainForm.ulWorkPtr := j;
Synchronize(MainThreadStatusRepaint);
end;
del := False;
if (j + 1) < links.Count then
for k := j + 1 to links.Count - 1 do
begin
if Terminated then Break;
if SameText(links[j], links[k]) then
begin
links.Delete(j);
names.Delete(j);
del := True;
Break;
end;
end;
if not del then
Inc(j);
end;
end;
tempDataProcess.OpenTable('', True);

// remove duplicate found<>current database
if (links.Count > 0) and (mainDataProcess.LinkCount > 0) then
begin
c := 0;
j := 0;
MainForm.ulTotalPtr := links.Count;
MainForm.ulWorkPtr := j;
if (mainDataProcess.LinkCount>0) and (tempDataProcess.RecordCount>0) then begin
MainForm.ulTotalPtr:=tempDataProcess.RecordCount;
MainForm.ulWorkPtr:=0;
FStatus := RS_UpdatingList + Format(' [%d/%d] %s',
[websitePtr, websites.Count, website]) + ' | ' + RS_RemovingDuplicateFromCurrentData + '...';
Synchronize(MainThreadShowGetting);
while j < links.Count do
begin
if Terminated then Break;
Inc(c);
if c > 999 then
begin
c := 0;
MainForm.ulTotalPtr := links.Count;
MainForm.ulWorkPtr := j;
Synchronize(MainThreadStatusRepaint);
with tempDataProcess.Table do begin
c:=0;
First;
while not tempDataProcess.Table.EOF do begin
if Terminated then Break;
Inc(c);
Inc(MainForm.ulWorkPtr);
if c>750 then begin
c:=0;
Synchronize(MainThreadStatusRepaint);
end;
if mainDataProcess.LinkExist(Fields[1].AsString) then
Delete
else
Next;
end;
if mainDataProcess.LinkExist(links[j]) then
begin
links.Delete(j);
names.Delete(j);
end
else
Inc(j);
ApplyUpdates;
end;
end;

tempDataProcess.Refresh(True);
mainDataProcess.DoneLocateLink;

//get manga info
if links.Count > 0 then
// get manga info
if tempDataProcess.RecordCount>0 then
begin
workPtr := 0;
FCommitCount := 0;
if NoMangaInfo or
OptionUpdateListNoMangaInfo then
begin
Inc(workPtr);
for k := 0 to links.Count - 1 do
for k:=0 to tempDataProcess.RecordCount-1 do
begin
mainDataProcess.AddData(
names[k],
links[k],
tempDataProcess.Value[k,0],
tempDataProcess.Value[k,1],
'',
'',
'',
Expand All @@ -723,12 +687,9 @@ procedure TUpdateMangaManagerThread.Execute;
end;
end
else
GetInfo(links.Count, CS_INFO);
GetInfo(tempDataProcess.RecordCount, CS_INFO);
mainDataProcess.Commit;

names.Clear;
links.Clear;

if workPtr > 0 then
if not (Terminated and SortedList) then
begin
Expand Down

0 comments on commit 148c3a6

Please sign in to comment.