--- /dev/null
+// -*- mode: cpp; mode: fold -*-
+// Description /*{{{*/
+// $Id: genfilelist.cc,v 1.10 1999/12/26 06:59:01 jgg Exp $
+/* ######################################################################
+
+ Generate File List
+
+ File list generation can be done with modification to the generation
+ order, ordering can be done by depth, breadth or by tree with and
+ a fitler can be applied to delay a directory till the end of processing.
+
+ The emitter simply generates the necessary structure and writes it to
+ the IO. The client can hook some of the functions to provide progress
+ reporting and md5 caching if so desired.
+
+ ##################################################################### */
+ /*}}}*/
+// Include files /*{{{*/
+#ifdef __GNUG__
+#pragma implementation "dsync/genfilelist.h"
+#endif
+
+#include <dsync/genfilelist.h>
+#include <dsync/error.h>
+#include <dsync/fileutl.h>
+#include <dsync/md5.h>
+#include <dsync/fileutl.h>
+#include <dsync/rsync-algo.h>
+
+#include <sys/stat.h>
+#include <unistd.h>
+#include <dirent.h>
+#include <stdio.h>
+ /*}}}*/
+
+// GenFileList::dsGenFileList - Constructor /*{{{*/
+// ---------------------------------------------------------------------
+/* */
+dsGenFileList::dsGenFileList() : IO(0), Type(Tree)
+{
+}
+ /*}}}*/
+// GenFileList::~dsGenFileList - Destructor /*{{{*/
+// ---------------------------------------------------------------------
+/* */
+dsGenFileList::~dsGenFileList()
+{
+}
+ /*}}}*/
+// GenFileList::Go - Generate the list /*{{{*/
+// ---------------------------------------------------------------------
+/* This invokes the proper recursive directory scanner to build the file
+ names. Depth and Breath use a queue */
+bool dsGenFileList::Go(string Base,dsFList::IO &IO)
+{
+ // Setup the queues and store the current directory
+ string StartDir = SafeGetCWD();
+ Queue.erase(Queue.begin(),Queue.end());
+ DelayQueue.erase(Queue.begin(),Queue.end());
+
+ struct stat St;
+ if (stat(Base.c_str(),&St) != 0)
+ return _error->Errno("stat","Could not stat the base directory");
+
+ // Begin
+ this->IO = &IO;
+ IO.Header.Write(IO);
+
+ switch (Type)
+ {
+ case Depth:
+ {
+ // Change to the base directory
+ if (chdir(Base.c_str()) != 0)
+ return _error->Errno("chdir","Could not change to %s",Base.c_str());
+ Base = SafeGetCWD();
+
+ char Cwd[1024];
+ Cwd[0] = 0;
+ if (DirDepthFirst(Cwd) == false)
+ {
+ chdir(StartDir.c_str());
+ return false;
+ }
+
+ // Now deal with the delay list
+ while (DelayQueue.empty() == false)
+ {
+ // Get the first delayed directory
+ string Dir = DelayQueue.front();
+ DelayQueue.pop_front();
+
+ // Change to it and emit it.
+ strcpy(Cwd,Dir.c_str());
+ chdir(Base.c_str());
+ chdir(Cwd);
+ if (DirDepthFirst(Cwd) == false)
+ {
+ chdir(StartDir.c_str());
+ return false;
+ }
+ }
+
+ break;
+ }
+
+ case Tree:
+ case Breadth:
+ {
+ // Change to the base directory
+ if (chdir(Base.c_str()) != 0)
+ return _error->Errno("chdir","Could not change to %s",Base.c_str());
+ Base = SafeGetCWD();
+
+ Queue.push_back("");
+ while (Queue.empty() == false || DelayQueue.empty() == false)
+ {
+ if (DirTree() == false)
+ {
+ chdir(StartDir.c_str());
+ return false;
+ }
+
+ chdir(Base.c_str());
+ }
+ break;
+ }
+
+ default:
+ return _error->Error("Internal Error");
+ };
+
+ chdir(StartDir.c_str());
+
+ dsFList::Trailer Trail;
+ return Trail.Write(IO);
+}
+ /*}}}*/
+// GenFileList::DirDepthFirst - Depth first directory ordering /*{{{*/
+// ---------------------------------------------------------------------
+/* */
+bool dsGenFileList::DirDepthFirst(char *CurDir)
+{
+ // Scan the directory, first pass is to descend into the sub directories
+ DIR *DirSt = opendir(".");
+ if (DirSt == 0)
+ return _error->Errno("opendir","Unable to open direcotry %s",CurDir);
+ struct dirent *Ent;
+ bool EmittedThis = false;
+ struct stat St;
+ while ((Ent = readdir(DirSt)) != 0)
+ {
+ // Skip . and ..
+ if (strcmp(Ent->d_name,".") == 0 ||
+ strcmp(Ent->d_name,"..") == 0)
+ continue;
+
+ if (lstat(Ent->d_name,&St) != 0)
+ {
+ closedir(DirSt);
+ return _error->Errno("stat","Could not stat %s%s",CurDir,Ent->d_name);
+ }
+
+ // it is a directory
+ if (S_ISDIR(St.st_mode) != 0)
+ {
+ char S[1024];
+ snprintf(S,sizeof(S),"%s/",Ent->d_name);
+
+ // Check the Filter
+ if (Filter.Test(CurDir,S) == false)
+ continue;
+
+ // Emit a directory marker record for this directory
+ if (EmittedThis == false)
+ {
+ EmittedThis = true;
+
+ if (lstat(".",&St) != 0)
+ {
+ closedir(DirSt);
+ return _error->Errno("stat","Could not stat %s",CurDir);
+ }
+
+ if (DirectoryMarker(CurDir,St) == false)
+ {
+ closedir(DirSt);
+ return false;
+ }
+ }
+
+ // Check the delay filter
+ if (PreferFilter.Test(CurDir,S) == false)
+ {
+ snprintf(S,sizeof(S),"%s%s/",CurDir,Ent->d_name);
+ DelayQueue.push_back(S);
+ continue;
+ }
+
+ // Append the new directory to CurDir and decend
+ char *End = CurDir + strlen(CurDir);
+ strcat(End,S);
+ if (chdir(S) != 0)
+ {
+ closedir(DirSt);
+ return _error->Errno("chdir","Could not chdir to %s%s",CurDir,S);
+ }
+
+ // Recurse
+ if (DirDepthFirst(CurDir) == false)
+ {
+ closedir(DirSt);
+ return false;
+ }
+
+ if (chdir("..") != 0)
+ {
+ closedir(DirSt);
+ return _error->Errno("chdir","Could not chdir to %s%s",CurDir,S);
+ }
+
+ // Chop off the directory we added to the current dir
+ *End = 0;
+ }
+ }
+ rewinddir(DirSt);
+
+ // Begin emitting this directory
+ if (lstat(".",&St) != 0)
+ {
+ closedir(DirSt);
+ return _error->Errno("stat","Could not stat %s",CurDir);
+ }
+
+ if (EnterDir(CurDir,St) == false)
+ {
+ closedir(DirSt);
+ return false;
+ }
+
+ while ((Ent = readdir(DirSt)) != 0)
+ {
+ // Skip . and ..
+ if (strcmp(Ent->d_name,".") == 0 ||
+ strcmp(Ent->d_name,"..") == 0)
+ continue;
+
+ struct stat St;
+ if (lstat(Ent->d_name,&St) != 0)
+ {
+ closedir(DirSt);
+ return _error->Errno("stat","Could not stat %s%s",CurDir,Ent->d_name);
+ }
+
+ // it is a directory
+ if (S_ISDIR(St.st_mode) != 0)
+ {
+ char S[1024];
+ snprintf(S,sizeof(S),"%s/",Ent->d_name);
+
+ // Check the Filter
+ if (Filter.Test(CurDir,S) == false)
+ continue;
+ }
+ else
+ {
+ // Check the Filter
+ if (Filter.Test(CurDir,Ent->d_name) == false)
+ continue;
+ }
+
+ if (DoFile(CurDir,Ent->d_name,St) == false)
+ {
+ closedir(DirSt);
+ return false;
+ }
+ }
+ closedir(DirSt);
+
+ if (LeaveDir(CurDir) == false)
+ return false;
+
+ return true;
+}
+ /*}}}*/
+// GenFileList::DirTree - Breadth/Tree directory ordering /*{{{*/
+// ---------------------------------------------------------------------
+/* Breadth ordering does all of the dirs at each depth before proceeding
+ to the next depth. We just treat the list as a queue to get this
+ effect. Tree ordering does things in a more normal recursive fashion,
+ we treat the queue as a stack to get that effect. */
+bool dsGenFileList::DirTree()
+{
+ string Dir;
+ if (Queue.empty() == false)
+ {
+ Dir = Queue.front();
+ Queue.pop_front();
+ }
+ else
+ {
+ Dir = DelayQueue.front();
+ DelayQueue.pop_front();
+ }
+
+ struct stat St;
+ if (Dir.empty() == false && chdir(Dir.c_str()) != 0 || stat(".",&St) != 0)
+ return _error->Errno("chdir","Could not change to %s",Dir.c_str());
+
+ if (EnterDir(Dir.c_str(),St) == false)
+ return false;
+
+ // Scan the directory
+ DIR *DirSt = opendir(".");
+ if (DirSt == 0)
+ return _error->Errno("opendir","Unable to open direcotry %s",Dir.c_str());
+ struct dirent *Ent;
+ while ((Ent = readdir(DirSt)) != 0)
+ {
+ // Skip . and ..
+ if (strcmp(Ent->d_name,".") == 0 ||
+ strcmp(Ent->d_name,"..") == 0)
+ continue;
+
+ if (lstat(Ent->d_name,&St) != 0)
+ {
+ closedir(DirSt);
+ return _error->Errno("stat","Could not stat %s%s",Dir.c_str(),Ent->d_name);
+ }
+
+ // It is a directory
+ if (S_ISDIR(St.st_mode) != 0)
+ {
+ char S[1024];
+ snprintf(S,sizeof(S),"%s/",Ent->d_name);
+
+ // Check the Filter
+ if (Filter.Test(Dir.c_str(),S) == false)
+ continue;
+
+ // Check the delay filter
+ if (PreferFilter.Test(Dir.c_str(),S) == false)
+ {
+ snprintf(S,sizeof(S),"%s%s/",Dir.c_str(),Ent->d_name);
+ if (Type == Tree)
+ DelayQueue.push_front(S);
+ else
+ DelayQueue.push_back(S);
+ continue;
+ }
+
+ snprintf(S,sizeof(S),"%s%s/",Dir.c_str(),Ent->d_name);
+
+ if (Type == Tree)
+ Queue.push_front(S);
+ else
+ Queue.push_back(S);
+ }
+ else
+ {
+ // Check the Filter
+ if (Filter.Test(Dir.c_str(),Ent->d_name) == false)
+ continue;
+ }
+
+ if (DoFile(Dir.c_str(),Ent->d_name,St) == false)
+ {
+ closedir(DirSt);
+ return false;
+ }
+ }
+ closedir(DirSt);
+
+ if (LeaveDir(Dir.c_str()) == false)
+ return false;
+
+ return true;
+}
+ /*}}}*/
+
+// GenFileList::EnterDir - Called when a directory is entered /*{{{*/
+// ---------------------------------------------------------------------
+/* This is called to start a directory block the current working dir
+ should be set to the directory entered. This emits the directory start
+ record */
+bool dsGenFileList::EnterDir(const char *Dir,struct stat const &St)
+{
+ if (Visit(Dir,0,St) != 0)
+ return false;
+
+ dsFList::Directory D;
+ D.Tag = dsFList::tDirStart;
+ D.ModTime = St.st_mtime - IO->Header.Epoch;
+ D.Permissions = St.st_mode & ~S_IFMT;
+ D.Name = Dir;
+ return EmitOwner(St,D.User,D.Group,D.Tag,dsFList::Directory::FlOwner) &&
+ D.Write(*IO);
+}
+ /*}}}*/
+// GenFileList::LeaveDir - Called when a directory is left /*{{{*/
+// ---------------------------------------------------------------------
+/* Don't do anything for now */
+bool dsGenFileList::LeaveDir(const char *Dir)
+{
+ return true;
+}
+ /*}}}*/
+// GenFileList::DirectoryMarker - Called when a dir is skipped /*{{{*/
+// ---------------------------------------------------------------------
+/* This is used by the depth first ordering, when a dir is temporarily
+ skipped over this function is called to emit a marker */
+bool dsGenFileList::DirectoryMarker(const char *Dir,
+ struct stat const &St)
+{
+ dsFList::Directory D;
+ D.Tag = dsFList::tDirMarker;
+ D.ModTime = St.st_mtime - IO->Header.Epoch;
+ D.Permissions = St.st_mode & ~S_IFMT;
+ D.Name = Dir;
+ return EmitOwner(St,D.User,D.Group,D.Tag,dsFList::Directory::FlOwner) &&
+ D.Write(*IO);
+}
+ /*}}}*/
+// GenFileList::DoFile - This does all other items in a directory /*{{{*/
+// ---------------------------------------------------------------------
+/* The different file types are emitted as perscribed by the file list
+ document */
+bool dsGenFileList::DoFile(const char *Dir,const char *File,
+ struct stat const &St)
+{
+ int Res = Visit(Dir,File,St);
+ if (Res < 0)
+ return false;
+ if (Res > 0)
+ return true;
+
+ // Regular file
+ if (S_ISREG(St.st_mode) != 0)
+ {
+ dsFList::NormalFile F;
+
+ F.Tag = dsFList::tNormalFile;
+ F.ModTime = St.st_mtime - IO->Header.Epoch;
+ F.Permissions = St.st_mode & ~S_IFMT;
+ F.Name = File;
+ F.Size = St.st_size;
+
+ if (EmitOwner(St,F.User,F.Group,F.Tag,dsFList::NormalFile::FlOwner) == false)
+ return false;
+
+ // See if we need to emit rsync checksums
+ if (NeedsRSync(Dir,File,F) == true)
+ {
+ dsFList::RSyncChecksum Ck;
+ if (EmitRSync(Dir,File,St,F,Ck) == false)
+ return false;
+
+ // Write out the file record, the checksums and the end marker
+ return F.Write(*IO) && Ck.Write(*IO);
+ }
+ else
+ {
+ if (EmitMD5(Dir,File,St,F.MD5,F.Tag,
+ dsFList::NormalFile::FlMD5) == false)
+ return false;
+
+ return F.Write(*IO);
+ }
+ }
+
+ // Directory
+ if (S_ISDIR(St.st_mode) != 0)
+ {
+ dsFList::Directory D;
+ D.Tag = dsFList::tDirectory;
+ D.ModTime = St.st_mtime - IO->Header.Epoch;
+ D.Permissions = St.st_mode & ~S_IFMT;
+ D.Name = File;
+ return EmitOwner(St,D.User,D.Group,D.Tag,dsFList::Directory::FlOwner) &&
+ D.Write(*IO);
+ }
+
+ // Link
+ if (S_ISLNK(St.st_mode) != 0)
+ {
+ dsFList::Symlink L;
+ L.Tag = dsFList::tSymlink;
+ L.ModTime = St.st_mtime - IO->Header.Epoch;
+ L.Name = File;
+
+ char Buf[1024];
+ int Res = readlink(File,Buf,sizeof(Buf));
+ if (Res <= 0)
+ return _error->Errno("readlink","Unable to read symbolic link");
+ Buf[Res] = 0;
+ L.To = Buf;
+
+ return EmitOwner(St,L.User,L.Group,L.Tag,dsFList::Symlink::FlOwner) &&
+ L.Write(*IO);
+ }
+
+ // Block special file
+ if (S_ISCHR(St.st_mode) != 0 || S_ISBLK(St.st_mode) != 0 ||
+ S_ISFIFO(St.st_mode) != 0)
+ {
+ dsFList::DeviceSpecial D;
+ D.Tag = dsFList::tDeviceSpecial;
+ D.ModTime = St.st_mtime - IO->Header.Epoch;
+ D.Permissions = St.st_mode & ~S_IFMT;
+ D.Dev = St.st_dev;
+ D.Name = File;
+
+ return EmitOwner(St,D.User,D.Group,D.Tag,dsFList::DeviceSpecial::FlOwner) &&
+ D.Write(*IO);
+ }
+
+ return _error->Error("File %s%s is not a known type",Dir,File);
+}
+ /*}}}*/
+// GenFileList::EmitOwner - Set the entitiy ownership /*{{{*/
+// ---------------------------------------------------------------------
+/* This emits the necessary UID/GID mapping records and sets the feilds
+ in */
+bool dsGenFileList::EmitOwner(struct stat const &St,unsigned long &UID,
+ unsigned long &GID,unsigned int Tag,
+ unsigned int Flag)
+{
+ if ((IO->Header.Flags[Tag] & Flag) != Flag)
+ return true;
+
+ return _error->Error("UID/GID storage is not supported yet");
+}
+ /*}}}*/
+// GenFileList::EmitMd5 - Generate the md5 hash for the file /*{{{*/
+// ---------------------------------------------------------------------
+/* This uses the MD5 class to generate the md5 hash for the entry. */
+bool dsGenFileList::EmitMD5(const char *Dir,const char *File,
+ struct stat const &St,unsigned char MD5[16],
+ unsigned int Tag,unsigned int Flag)
+{
+ if ((IO->Header.Flags[Tag] & Flag) != Flag)
+ return true;
+
+ // Open the file
+ MD5Summation Sum;
+ FileFd Fd(File,FileFd::ReadOnly);
+ if (_error->PendingError() == true)
+ return _error->Error("MD5 generation failed for %s%s",Dir,File);
+
+ if (Sum.AddFD(Fd.Fd(),Fd.Size()) == false)
+ return _error->Error("MD5 generation failed for %s%s",Dir,File);
+
+ Sum.Result().Value(MD5);
+
+ return true;
+}
+ /*}}}*/
+// GenFileList::EmitRSync - Emit a RSync checksum record /*{{{*/
+// ---------------------------------------------------------------------
+/* This just generates the checksum into the memory structure. */
+bool dsGenFileList::EmitRSync(const char *Dir,const char *File,
+ struct stat const &St,dsFList::NormalFile &F,
+ dsFList::RSyncChecksum &Ck)
+{
+ FileFd Fd(File,FileFd::ReadOnly);
+ if (_error->PendingError() == true)
+ return _error->Error("RSync Checksum generation failed for %s%s",Dir,File);
+
+ if (GenerateRSync(Fd,Ck,F.MD5) == false)
+ return _error->Error("RSync Checksum generation failed for %s%s",Dir,File);
+
+ return true;
+}
+ /*}}}*/
projects / dak.git / blobdiff