// @(#)root/proof:$Id$ // Author: G.Ganis, S.Ryu Feb 2011 /************************************************************************* * Copyright (C) 1995-2005, Rene Brun and Fons Rademakers. * * All rights reserved. * * * * For the licensing terms see $ROOTSYS/LICENSE. * * For the list of contributors see $ROOTSYS/README/CREDITS. * *************************************************************************/ ////////////////////////////////////////////////////////////////////////// // // // TProofBench // // // // Steering class for PROOF benchmarks // // // ////////////////////////////////////////////////////////////////////////// #include "RConfigure.h" #include "TProofBench.h" #include "Getline.h" #include "TProofBenchRunCPU.h" #include "TProofBenchRunDataRead.h" #include "TProofBenchDataSet.h" #include "TProofNodes.h" #include "TClass.h" #include "TFile.h" #include "TFileCollection.h" #include "TFileInfo.h" #include "THashList.h" #include "TKey.h" #include "TObjString.h" #include "TProof.h" #include "TROOT.h" #include "TSortedList.h" #include "TTimeStamp.h" #include "TUrl.h" #include "TCanvas.h" #include "TF1.h" #include "TGraphErrors.h" #include "TH1F.h" #include "TMath.h" #include "TProfile.h" #include "TStyle.h" #include "TLegend.h" ClassImp(TProofBench) // Functions for fitting TF1 *TProofBench::fgFp1 = 0; TF1 *TProofBench::fgFp1n = 0; TF1 *TProofBench::fgFp2 = 0; TF1 *TProofBench::fgFp2n = 0; TF1 *TProofBench::fgFp3 = 0; TF1 *TProofBench::fgFp3n = 0; TF1 *TProofBench::fgFio = 0; TF1 *TProofBench::fgFioV = 0; static Int_t gFioVn0 = -1; // Number of real cores for fgFioV static Int_t gFioVn1 = -1; // Number of real+hyper cores for fgFioV TList *TProofBench::fgGraphs = new TList; //_____________________________________________________________________ Double_t funp1(Double_t *xx, Double_t *par) { // Simple polynomial 1st degree Double_t res = par[0] + par[1] * xx[0]; return res; } //_____________________________________________________________________ Double_t funp2(Double_t *xx, Double_t *par) { // Simple polynomial 2nd degree Double_t res = par[0] + par[1] * xx[0] + par[2] * xx[0] * xx[0]; return res; } //_____________________________________________________________________ Double_t funp1n(Double_t *xx, Double_t *par) { // Normalized 1st degree Double_t res = par[0] / xx[0] + par[1]; return res; } //_____________________________________________________________________ Double_t funp2n(Double_t *xx, Double_t *par) { // Normalized 2nd degree Double_t res = par[0] / xx[0] + par[1] + par[2] * xx[0]; return res; } //_____________________________________________________________________ Double_t funio(Double_t *xx, Double_t *par) { // I/O saturated rate function Double_t sat = par[0] / par[1] * (xx[0] * par[1] / par[2] - 1.); if (xx[0] < par[2] / par[1]) sat = 0.; Double_t res = par[0] * xx[0] / (1. + sat); return res; } //_____________________________________________________________________ Double_t funiov(Double_t *xx, Double_t *par) { // I/O saturated rate function with varying Rcpu // par[0] = rio // par[1] = b1 // par[2] = b2 // par[3] = nc // par[4] = ri Double_t rio = par[0] / par[3] * xx[0]; if (xx[0] > par[3]) rio = par[0]; Double_t rcpu = par[1] * xx[0]; if (xx[0] > gFioVn0) rcpu = par[1]*gFioVn0 + par[2]*(xx[0] - gFioVn0); if (xx[0] > gFioVn1) rcpu = par[1]*gFioVn0 + par[2]*(gFioVn1 - gFioVn0); Double_t res = 1. / (1./par[4] + 1./rio + 1./rcpu); return res; } //_____________________________________________________________________ Double_t funcpuv(Double_t *xx, Double_t *par) { // Function with varying Rcpu // par[0] = offset // par[1] = rate contribution from real cores // par[2] = rate contribution from hyper cores Double_t n = (xx[0] - par[0]); Double_t rcpu = par[1] * n; if (xx[0] > gFioVn0) rcpu = par[1]*gFioVn0 + par[2]*(n - gFioVn0); if (xx[0] > gFioVn1) rcpu = par[1]*gFioVn0 + par[2]*(gFioVn1 - gFioVn0); return rcpu; } //_____________________________________________________________________ Double_t funcpuvn(Double_t *xx, Double_t *par) { // Function with varying Rcpu normalized // par[0] = offset // par[1] = rate contribution from real cores // par[2] = rate contribution from hyper cores Double_t n = (xx[0] - par[0]); Double_t rcpu = par[1] * n; if (xx[0] > gFioVn0) rcpu = par[1]*gFioVn0 + par[2]*(n - gFioVn0); if (xx[0] > gFioVn1) rcpu = par[1]*gFioVn0 + par[2]*(gFioVn1 - gFioVn0); return rcpu / xx[0]; } //______________________________________________________________________________ TProofBench::TProofBench(const char *url, const char *outfile, const char *proofopt) : fUnlinkOutfile(kFALSE), fProofDS(0), fOutFile(0), fNtries(4), fHistType(0), fNHist(16), fReadType(0), fDataSet("BenchDataSet"), fNFilesWrk(2), fReleaseCache(kTRUE), fDataGenSel(kPROOF_BenchSelDataGenDef), fRunCPU(0), fRunDS(0), fDS(0), fDebug(kFALSE), fDescription(0) { // Constructor: check PROOF and load selectors PAR SetBit(kInvalidObject); if (!url) { Error("TProofBench", "specifying a PROOF master url is mandatory - cannot continue"); return; } if (!(fProof = TProof::Open(url, proofopt)) || (fProof && !fProof->IsValid())) { Error("TProofBench", "could not open a valid PROOF session - cannot continue"); return; } // Get the size of the cluster fNumWrkMax = fProof->GetParallel(); if (fProof->UseDynamicStartup() && TProof::GetEnvVars()) { // It must be passed as PROOF option 'workers=N' and recorded in the envs vars TNamed *n = (TNamed *) TProof::GetEnvVars()->FindObject("PROOF_NWORKERS"); if (!n) { Error("TProofBench", "dynamic mode: you must specify the max number of workers"); fProof->Close(); SafeDelete(fProof); return; } TString sn(n->GetTitle()); if (sn.IsDigit()) fNumWrkMax = sn.Atoi(); if (!sn.IsDigit()) { Error("TProofBench", "dynamic mode: wrong specification of the max number of" " workers ('%s')", n->GetTitle()); fProof->Close(); SafeDelete(fProof); return; } } if (fNumWrkMax <= 0) { Error("TProofBench", "wrong max number of workers ('%d')", fNumWrkMax); fProof->Close(); SafeDelete(fProof); return; } // By default we use the same instance for dataset actions fProofDS = fProof; // The object is now valid ResetBit(kInvalidObject); // Identifying string TUrl u(url); TString host(TString::Format("PROOF at %s", u.GetHost())); if (!strcmp(u.GetProtocol(), "lite")) host.Form("PROOF-Lite on %s", gSystem->HostName()); fDescription = new TNamed("PB_description", TString::Format("%s, %d workers", host.Data(), fNumWrkMax).Data()); Printf(" Run description: %s", fDescription->GetTitle()); // Set output file if (SetOutFile(outfile, kFALSE) != 0) Warning("TProofBench", "problems opening '%s' - ignoring: use SetOutFile to try" " again or with another file", outfile); } //______________________________________________________________________________ TProofBench::~TProofBench() { // Destructor CloseOutFile(); if (fUnlinkOutfile) gSystem->Unlink(fOutFileName); SafeDelete(fReadType); SafeDelete(fRunCPU); SafeDelete(fRunDS); SafeDelete(fDescription); } //______________________________________________________________________________ Int_t TProofBench::OpenOutFile(Bool_t wrt, Bool_t verbose) { // Set the otuput file // Return 0 on success, -1 on error // Remove any bad file if (fOutFile && fOutFile->IsZombie()) SafeDelete(fOutFile); Int_t rc = 0; if (!fOutFile && fOutFileName.Length() > 0) { const char *mode = 0; if (wrt) mode = gSystem->AccessPathName(fOutFileName) ? "RECREATE" : "UPDATE"; else mode = "READ"; if (!(fOutFile = TFile::Open(fOutFileName, mode)) || (fOutFile && fOutFile->IsZombie())) { if (verbose) Warning("OpenOutFile", "problems opening '%s' - ignoring: use SetOutFile to try" " again or with another file", fOutFileName.Data()); rc = -1; } if (fOutFile) { gROOT->GetListOfFiles()->Remove(fOutFile); if (!strcmp(mode, "RECREATE")) { // Save the description string fOutFile->cd(); fDescription->Write(); } } } return rc; } //______________________________________________________________________________ Int_t TProofBench::SetOutFile(const char *outfile, Bool_t verbose) { // Set the output file // Return 0 on success, -1 on error Int_t rc = 0; // Close existing file, if any if (fOutFile) { if (!fOutFile->IsZombie()) fOutFile->Close(); SafeDelete(fOutFile); } fOutFileName = outfile; if (fOutFileName == "") { // Default output file: proofbench--.root TDatime dat; const char *lite = (fProof->IsLite()) ? "-lite" : ""; fOutFileName.Form("proofbench-%s%s-%dw-%d-%.2d%.2d.root", fProof->GetMaster(), lite, fNumWrkMax, dat.GetDate(), dat.GetHour(), dat.GetMinute()); Info("SetOutFile", "using default output file: '%s'", fOutFileName.Data()); fUnlinkOutfile = kTRUE; } if (!fOutFileName.IsNull()) { if ((rc = OpenOutFile(kTRUE, kFALSE)) != 0 && verbose) Warning("SetOutFile", "problems opening '%s' - ignoring: use SetOutFile to try" " again or with another file", outfile); } return rc; } //______________________________________________________________________________ void TProofBench::CloseOutFile() { // Close output file if (SetOutFile(0) != 0) Warning("CloseOutFile", "problems closing '%s'", fOutFileName.Data()); } //______________________________________________________________________________ Int_t TProofBench::RunCPU(Long64_t nevents, Int_t start, Int_t stop, Int_t step) { // Perform the CPU run // Return 0 on success, -1 on error // Open the file for the results if (OpenOutFile(kTRUE) != 0) { Error("RunCPU", "problems opening '%s' to save the result", fOutFileName.Data()); return -1; } fUnlinkOutfile = kFALSE; SafeDelete(fRunCPU); TPBHistType htype(TPBHistType::kHist1D); fRunCPU = new TProofBenchRunCPU(&htype, fNHist, fOutFile); if (!fCPUSel.IsNull()) fRunCPU->SetSelName(fCPUSel); if (!fCPUPar.IsNull()) fRunCPU->SetParList(fCPUPar); fRunCPU->Run(nevents, start, stop, step, fNtries, fDebug, -1); // Close the file if (SetOutFile(0) != 0) Warning("RunCPU", "problems closing '%s'", fOutFileName.Data()); // Done return 0; } //______________________________________________________________________________ Int_t TProofBench::RunCPUx(Long64_t nevents, Int_t start, Int_t stop) { // Perform the CPU run scanning over the number of workers per node // Return 0 on success, -1 on error // Open the file for the results if (OpenOutFile(kTRUE) != 0) { Error("RunCPUx", "problems opening '%s' to save the result", fOutFileName.Data()); return -1; } fUnlinkOutfile = kFALSE; SafeDelete(fRunCPU); TPBHistType htype(TPBHistType::kHist1D); fRunCPU = new TProofBenchRunCPU(&htype, fNHist, fOutFile); if (!fCPUSel.IsNull()) fRunCPU->SetSelName(fCPUSel); if (!fCPUPar.IsNull()) fRunCPU->SetParList(fCPUPar); fRunCPU->Run(nevents, start, stop, -2, fNtries, fDebug, -1); // Close the file if (SetOutFile(0) != 0) Warning("RunCPUx", "problems closing '%s'", fOutFileName.Data()); // Done return 0; } //______________________________________________________________________________ void TProofBench::DrawCPU(const char *outfile, const char *opt, Bool_t verbose, Int_t dofit, Int_t n0, Int_t n1) { // Draw the CPU speedup plot. // opt = 'typewhat', e.g. 'std:max:' // type = 'std:' draw standard evt/s plot // 'stdx:' draw standard evt/s plot, 1 worker per node // 'norm:' draw normalized plot // 'normx:' draw normalized plot, 1 worker per node // what = 'max:' draw max rate // 'avg:' draw average rate // 'all:' draw max and average rate on same plot (default) // dofit = 0 no fit // 1 fit with the relevant '1st degree related' function // 2 fit with the relevant '2nd degree related' function // 3 fit with varying rcpu function // n0 = for dofit == 3, number of real cores // n1 = for dofit == 3, number of total cores (real + hyperthreaded) // // Get the TProfile an create the graphs TFile *fout = TFile::Open(outfile, "READ"); if (!fout || (fout && fout->IsZombie())) { ::Error("DrawCPU", "could not open file '%s' ...", outfile); return; } // Get description TString description(""); TNamed *nmdesc = (TNamed *) fout->Get("PB_description"); if (nmdesc) description = nmdesc->GetTitle(); // Parse option TString oo(opt); Bool_t isNorm = (oo.Contains("norm")) ? kTRUE : kFALSE; Bool_t isX = (oo.Contains("stdx:") || oo.Contains("normx:")) ? kTRUE : kFALSE; Bool_t doAvg = kTRUE, doMax = kTRUE; if (oo.Contains("avg:")) doMax = kFALSE; if (oo.Contains("max:")) doAvg = kFALSE; const char *dirn = (isX) ? "RunCPUx" : "RunCPU"; TDirectory *d = (TDirectory *) fout->Get(dirn); if (!d) { ::Error("DrawCPU", "could not find directory '%s' ...", dirn); fout->Close(); delete fout; return; } d->cd(); TString hprofn, hmaxn; const char *lx = (isX) ? "_x" : ""; const char *ln = (isNorm) ? "Norm" : "Prof"; hprofn.Form("%s%s_CPU_QR_Evts", ln, lx); hmaxn.Form("%s%s_CPU_PS_MaxEvts", ln, lx); Double_t xmin = -1., xmax = -1.; Double_t ami = -1., amx = -1., mmi = -1., mmx = -1.; Int_t kamx = -1, kmmx = -1, nbins = -1; Double_t ymx = -1., ymi = -1.; TProfile *pfav = 0; TGraphErrors *grav = 0; if (doAvg) { if (!(grav = GetGraph(d, hprofn, nbins, xmin, xmax, ami, amx, kamx, pfav))) { ::Error("DrawCPU", "could not find '%s' ...", hprofn.Data()); fout->Close(); delete fout; return; } ymx = amx; ymi = ami; } TProfile *pfmx = 0; TGraphErrors *grmx = 0; if (doMax) { if (!(grmx = GetGraph(d, hmaxn, nbins, xmin, xmax, mmi, mmx, kmmx, pfmx))) { ::Warning("DrawCPU", "could not find '%s': feature added in 5.34/11", hmaxn.Data()); if (!grav) { // Nothing to do if not asked for the average fout->Close(); delete fout; return; } doMax = kFALSE; } if (mmx > ymx) ymx = mmx; if ((ymi > 0 && mmi < ymi) || (ymi < 0.)) ymi = mmi; } TProfile *pf = (doMax) ? pfmx : pfav; Int_t kmx = (doMax) ? kmmx : kamx; // Create the canvas TCanvas *cpu = new TCanvas("cpu", "Rate vs wrks",204,69,1050,502); cpu->Range(-3.106332,0.7490716,28.1362,1.249867); TH1F *hgr = new TH1F("Graph-CPU"," CPU speed-up", nbins*4, xmin, xmax); hgr->SetMaximum(ymx + (ymx-ymi)*0.2); hgr->SetMinimum(0); if (isNorm) hgr->SetMaximum(ymx*1.2); hgr->SetDirectory(0); hgr->SetStats(0); hgr->GetXaxis()->SetTitle(pf->GetXaxis()->GetTitle()); hgr->GetXaxis()->CenterTitle(true); hgr->GetXaxis()->SetLabelSize(0.05); hgr->GetXaxis()->SetTitleSize(0.06); hgr->GetXaxis()->SetTitleOffset(0.62); hgr->GetYaxis()->SetTitleSize(0.08); hgr->GetYaxis()->SetTitleOffset(0.52); hgr->GetYaxis()->SetTitle("Rate (events/s)"); TLegend *leg = 0; if (isNorm) { leg = new TLegend(0.7, 0.8, 0.9, 0.9); } else { leg = new TLegend(0.1, 0.8, 0.3, 0.9); } gStyle->SetOptTitle(0); if (doAvg) { grav->SetFillColor(1); grav->SetLineColor(13); grav->SetMarkerColor(4); grav->SetMarkerStyle(21); grav->SetMarkerSize(1.2); grav->SetHistogram(hgr); if (verbose) grav->Print(); grav->Draw("alp"); leg->AddEntry(grav, "Average", "P"); } if (doMax) { grmx->SetFillColor(1); grmx->SetLineColor(13); grmx->SetMarkerColor(2); grmx->SetMarkerStyle(29); grmx->SetMarkerSize(1.8); grmx->SetHistogram(hgr); if (verbose) grmx->Print(); if (doAvg) { grmx->Draw("lpSAME"); } else { grmx->Draw("alp"); } leg->AddEntry(grmx, "Maximum", "P"); } leg->Draw(); gPad->Update(); if (dofit > 0) { TGraphErrors *gr = (doMax) ? grmx : grav; // Make sure the fitting functions are defined Double_t xmi = 0.9; if (nbins > 5) xmi = 1.5; AssertFittingFun(xmi, nbins + .1); // Starting point for the parameters and fit Double_t normrate = -1.; if (dofit == 1) { if (isNorm) { fgFp1n->SetParameter(0, pf->GetBinContent(1)); fgFp1n->SetParameter(1, pf->GetBinContent(nbins-1)); gr->Fit(fgFp1n); if (verbose) fgFp1n->Print(); normrate = fgFp1n->GetParameter(1); } else { fgFp1->SetParameter(0, 0.); fgFp1->SetParameter(1, pf->GetBinContent(1)); gr->Fit(fgFp1); if (verbose) fgFp1->Print(); normrate = fgFp1->Derivative(1.); } } else if (dofit == 2) { if (isNorm) { fgFp2n->SetParameter(0, pf->GetBinContent(1)); fgFp2n->SetParameter(1, pf->GetBinContent(nbins-1)); fgFp2n->SetParameter(2, 0.); gr->Fit(fgFp2n); if (verbose) fgFp2n->Print(); normrate = fgFp2n->GetParameter(1); } else { fgFp2->SetParameter(0, 0.); fgFp2->SetParameter(1, pf->GetBinContent(1)); fgFp2->SetParameter(2, 0.); gr->Fit(fgFp2); if (verbose) fgFp2->Print(); normrate = fgFp2->Derivative(1.); } } else { // Starting point for the parameters and fit gFioVn0 = (n0 > 0) ? n0 : (Int_t) (nbins + .1)/2.; gFioVn1 = (n1 > 0) ? n1 : (Int_t) (nbins + .1); if (isNorm) { fgFp3n->SetParameter(0, 0.); fgFp3n->SetParameter(1, pf->GetBinContent(1)); fgFp3n->SetParameter(2, pf->GetBinContent(nbins-1)); gr->Fit(fgFp3n); if (verbose) fgFp3n->Print(); normrate = pf->GetBinContent(1); } else { fgFp3->SetParameter(0, 0.); fgFp3->SetParameter(1, 0.); fgFp3->SetParameter(2, pf->GetBinContent(1)); gr->Fit(fgFp3); if (verbose) fgFp3->Print(); normrate = fgFp3->Derivative(1.); } } // Notify the cluster performance parameters if (!isNorm) { printf("* ************************************************************ *\n"); printf("* *\r"); printf("* Cluster: %s\n", description.Data()); printf("* Performance measurement from scalability plot: *\n"); printf("* *\r"); printf("* rate max: %.3f\tmegaRNGPS (@ %d workers)\n", ymx/1000000, kmx); printf("* *\r"); printf("* per-worker rate: %.3f\tmegaRNGPS \n", normrate/1000000); printf("* ************************************************************ *\n"); } else { printf("* ************************************************************ *\n"); printf("* *\r"); printf("* Cluster: %s\n", description.Data()); printf("* *\r"); printf("* Per-worker rate from normalized plot: %.3f\tmegaRNGPS\n", normrate/1000000); printf("* ************************************************************ *\n"); } } // Close the file fout->Close(); if (grav) fgGraphs->Add(grav); if (grmx) fgGraphs->Add(grmx); } //______________________________________________________________________________ TGraphErrors *TProofBench::GetGraph(TDirectory *d, const char *pfn, Int_t &nb, Double_t &xmi, Double_t &xmx, Double_t &ymi, Double_t &ymx, Int_t &kmx, TProfile *&pf) { // Get from TDirectory 'd' the TProfile named 'pfn' and create the graph. // Return also the max y in mx. // Sanity checks if (!d || !pfn || (pfn && strlen(pfn) <= 0)) { ::Error("TProofBench::GetGraph", "directory or name not defined!"); return (TGraphErrors *)0; } TList *keylist = d->GetListOfKeys(); TKey *key = 0; TIter nxk(keylist); while ((key = (TKey *) nxk())) { if (TString(key->GetName()).BeginsWith(pfn)) { pf = (TProfile *) d->Get(key->GetName()); break; } } // Sanity checks if (!pf) { ::Error("TProofBench::GetGraph", "TProfile for '%s' not found in directory '%s'", pfn, d->GetName()); return (TGraphErrors *)0; } nb = pf->GetNbinsX(); TGraphErrors *gr = new TGraphErrors(nb); gr->SetName(TString::Format("Graph_%s", pfn)); Double_t xx, ex, yy, ey; ymi = pf->GetBinContent(1); ymx = ymi; xmi = pf->GetBinCenter(1) - pf->GetBinWidth(1)/2. ; xmx = pf->GetBinCenter(nb) + pf->GetBinWidth(nb)/2. ; kmx = -1; for (Int_t k = 1;k <= nb; k++) { xx = pf->GetBinCenter(k); ex = pf->GetBinWidth(k) * .001; yy = pf->GetBinContent(k); ey = pf->GetBinError(k); if (k == 1) { ymi = yy; ymx = yy; kmx = k; } else { if (yy < ymi) ymi = yy; if (yy > ymx) { ymx = yy; kmx = k; } } gr->SetPoint(k-1, xx, yy); gr->SetPointError(k-1, ex, ey); } // Done return gr; } //______________________________________________________________________________ void TProofBench::AssertFittingFun(Double_t mi, Double_t mx) { // Make sure that the fitting functions are defined if (!fgFp1) { fgFp1 = new TF1("funp1", funp1, mi, mx, 2); fgFp1->SetParNames("offset", "slope"); } if (!fgFp1n) { fgFp1n = new TF1("funp1n", funp1n, mi, mx, 2); fgFp1n->SetParNames("decay", "norm rate"); } if (!fgFp2) { fgFp2 = new TF1("funp2", funp2, mi, mx, 3); fgFp2->SetParNames("offset", "slope", "deviation"); } if (!fgFp2n) { fgFp2n = new TF1("funp2n", funp2n, mi, mx, 3); fgFp2n->SetParNames("decay", "norm rate", "deviation"); } if (!fgFp3) { fgFp3 = new TF1("funcpuv", funcpuv, mi, mx, 3); fgFp3->SetParNames("offset", "slope real", "slope hyper"); } if (!fgFp3n) { fgFp3n = new TF1("funcpuvn", funcpuvn, mi, mx, 3); fgFp3n->SetParNames("offset", "slope real", "slope hyper"); } if (!fgFio) { fgFio = new TF1("funio", funio, mi, mx, 3); fgFio->SetParNames("R1", "RIO", "TotIO"); } if (!fgFioV) { fgFioV = new TF1("funiov", funiov, mi, mx, 5); fgFioV->SetParNames("rio", "b1", "b2", "nc", "ri"); } } //______________________________________________________________________________ class fileDesc : public TNamed { public: Long_t fMtime; // Modification time TString fDesc; // Test description string, if any fileDesc(const char *n, const char *o, Long_t t, const char *d) : TNamed(n, o), fMtime(t), fDesc(d) { } Int_t Compare(const TObject *o) const { const fileDesc *fd = static_cast(o); if (!fd || (fd && fd->fMtime == fMtime)) return 0; if (fMtime < fd->fMtime) return -1; return 1; } }; //______________________________________________________________________________ void TProofBench::GetPerfSpecs(const char *path, Int_t degfit) { // Get performance specs. Check file 'path', or files in directory 'path' // (default current directory). // The degree of the polynomial used for the fit is 'degfit' (default 1). // Locate the file (ask if many) TString pp(path), fn, oo; if (pp.IsNull()) pp = gSystem->WorkingDirectory(); FileStat_t st; if (gSystem->GetPathInfo(pp.Data(), st) != 0) { ::Error("TProofBench::GetPerfSpecs", "path '%s' could not be stat'ed - abort", pp.Data()); return; } TSortedList filels; if (R_ISDIR(st.fMode)) { // Scan the directory void *dirp = gSystem->OpenDirectory(pp.Data()); if (!dirp) { ::Error("TProofBench::GetPerfSpecs", "directory path '%s' could nto be open - abort", pp.Data()); return; } const char *ent = 0; while ((ent = gSystem->GetDirEntry(dirp))) { if (!strcmp(ent, ".") || !strcmp(ent, "..")) continue; fn.Form("%s/%s", pp.Data(), ent); if (gSystem->GetPathInfo(fn.Data(), st) != 0) continue; if (!R_ISREG(st.fMode)) continue; fn += "?filetype=raw"; TFile *f = TFile::Open(fn); if (!f) continue; char rr[5] = {0}; if (!f->ReadBuffer(rr, 4)) { if (!strncmp(rr, "root", 4)) { SafeDelete(f); fn.ReplaceAll("?filetype=raw", ""); if ((f = TFile::Open(fn))) { TString desc(""); TNamed *nmdesc = (TNamed *) f->Get("PB_description"); if (nmdesc) desc = nmdesc->GetTitle(); if (f->GetListOfKeys()->FindObject("RunCPU")) filels.Add(new fileDesc(fn, "std:", st.fMtime, desc.Data())); if (f->GetListOfKeys()->FindObject("RunCPUx")) filels.Add(new fileDesc(fn, "stdx:", st.fMtime, desc.Data())); } else { ::Warning("TProofBench::GetPerfSpecs", "problems opening '%s'", fn.Data()); } } } SafeDelete(f); } } else if (!R_ISREG(st.fMode)) { ::Error("TProofBench::GetPerfSpecs", "path '%s' not a regular file nor a directory - abort", pp.Data()); return; } else { // This is the file fn = pp; // Check it TString emsg; Bool_t isOk = kFALSE; if (gSystem->GetPathInfo(fn.Data(), st) == 0) { fn += "?filetype=raw"; TFile *f = TFile::Open(fn); if (f) { char rr[5] = {0}; if (!(f->ReadBuffer(rr, 4))) { if (!strncmp(rr, "root", 4)) { fn.ReplaceAll("?filetype=raw", ""); if ((f = TFile::Open(fn))) { if (f->GetListOfKeys()->FindObject("RunCPU")) oo = "std:"; if (f->GetListOfKeys()->FindObject("RunCPUx")) oo = "stdx:"; SafeDelete(f); if (!oo.IsNull()) { isOk = kTRUE; } else { emsg.Form("path '%s' does not contain the relevant dirs - abort", fn.Data()); } } else { emsg.Form("path '%s' cannot be open - abort", fn.Data()); } } else { emsg.Form("'%s' is not a ROOT file - abort", fn.Data()); } } else { emsg.Form("could not read first 4 bytes from '%s' - abort", fn.Data()); } SafeDelete(f); } else { emsg.Form("path '%s' cannot be open in raw mode - abort", fn.Data()); } } else { emsg.Form("path '%s' cannot be stated - abort", fn.Data()); } if (!isOk) { ::Error("TProofBench::GetPerfSpecs", "%s", emsg.Data()); return; } } fileDesc *nm = 0; // Ask the user, if more then 1 if (filels.GetSize() == 1) { nm = (fileDesc *) filels.First(); fn = nm->GetName(); oo = nm->GetTitle(); } else if (filels.GetSize() > 1) { TIter nxf(&filels); Int_t idx = 0; Printf("Several possible files found:"); while ((nm = (fileDesc *) nxf())) { Printf(" %d\t%s\t%s\t%s (file: %s)", idx++, nm->GetTitle(), TTimeStamp(nm->fMtime).AsString("s"), nm->fDesc.Data(), nm->GetName()); } TString a(Getline(TString::Format("Make your choice [%d] ", idx-1))); if (a.IsNull() || a[0] == '\n') a.Form("%d", idx-1); idx = a.Atoi(); if ((nm = (fileDesc *) filels.At(idx))) { fn = nm->GetName(); oo = nm->GetTitle(); } else { ::Error("TProofBench::GetPerfSpecs", "chosen index '%d' does not exist - abort", idx); return; } } else { if (fn.IsNull()) { ::Error("TProofBench::GetPerfSpecs", "path '%s' is a directory but no ROOT file found in it - abort", pp.Data()); return; } } // Now get the specs TProofBench::DrawCPU(fn.Data(), oo.Data(), kFALSE, degfit); } //______________________________________________________________________________ Int_t TProofBench::RunDataSet(const char *dset, Int_t start, Int_t stop, Int_t step) { // Perform a test using dataset 'dset' // Return 0 on success, -1 on error // Open the file for the results if (OpenOutFile(kTRUE) != 0) { Error("RunDataSet", "problems opening '%s' to save the result", fOutFileName.Data()); return -1; } fUnlinkOutfile = kFALSE; if (fReleaseCache) ReleaseCache(dset); SafeDelete(fRunDS); TPBReadType *readType = fReadType; if (!readType) readType = new TPBReadType(TPBReadType::kReadOpt); fRunDS = new TProofBenchRunDataRead(fDS, readType, fOutFile); if (!fDataSel.IsNull()) fRunDS->SetSelName(fDataSel); if (!fDataPar.IsNull()) fRunDS->SetParList(fDataPar); fRunDS->SetReleaseCache(fReleaseCache); fRunDS->Run(dset, start, stop, step, fNtries, fDebug, -1); if (!fReadType) SafeDelete(readType); // Close the file if (SetOutFile(0) != 0) Warning("RunDataSet", "problems closing '%s'", fOutFileName.Data()); // Done return 0; } //______________________________________________________________________________ Int_t TProofBench::RunDataSetx(const char *dset, Int_t start, Int_t stop) { // Perform a test using dataset 'dset' scanning over the number of workers // per node. // Return 0 on success, -1 on error // Open the file for the results if (OpenOutFile(kTRUE) != 0) { Error("RunDataSetx", "problems opening '%s' to save the result", fOutFileName.Data()); return -1; } fUnlinkOutfile = kFALSE; ReleaseCache(dset); SafeDelete(fRunDS); TPBReadType *readType = fReadType; if (!readType) readType = new TPBReadType(TPBReadType::kReadOpt); fRunDS = new TProofBenchRunDataRead(fDS, readType, fOutFile); if (!fDataSel.IsNull()) fRunDS->SetSelName(fDataSel); if (!fDataPar.IsNull()) fRunDS->SetParList(fDataPar); fRunDS->Run(dset, start, stop, -2, fNtries, fDebug, -1); if (!fReadType) SafeDelete(readType); // Close the file if (SetOutFile(0) != 0) Warning("RunDataSetx", "problems closing '%s'", fOutFileName.Data()); // Done return 0; } //______________________________________________________________________________ void TProofBench::DrawDataSet(const char *outfile, const char *opt, const char *type, Bool_t verbose, Int_t dofit, Int_t n0, Int_t n1) { // Draw the CPU speedup plot. // opt = 'typewhat', e.g. 'std:max:' // type = 'std:' draw standard plot // 'stdx:' draw standard plot, 1 worker per node // 'norm:' draw normalized plot // 'normx:' draw normalized plot, 1 worker per node // what = 'max:' draw max rate // 'avg:' draw average rate // 'all:' draw max and average rate on same plot (default) // type = 'mbs' MB/s scaling plots (default) // 'evts' Event/s scaling plots // dofit = 0 no fit // 1 fit with default 3 parameter saturated I/O formula // 2 fit with 4 parameter saturated I/O formula (varying Rcpu) // n0 = for dofit == 2, number of real cores // n1 = for dofit == 2, number of total cores (real + hyperthreaded) // // Get the TProfile an create the graphs TFile *fout = TFile::Open(outfile, "READ"); if (!fout || (fout && fout->IsZombie())) { ::Error("DrawDataSet", "could not open file '%s' ...", outfile); return; } // Get description TString description(""); TNamed *nmdesc = (TNamed *) fout->Get("PB_description"); if (nmdesc) description = nmdesc->GetTitle(); // Parse option TString oo(opt); Bool_t isNorm = (oo.Contains("norm")) ? kTRUE : kFALSE; Bool_t isX = (oo.Contains("stdx:") || oo.Contains("normx:")) ? kTRUE : kFALSE; Bool_t doAvg = kTRUE, doMax = kTRUE; if (oo.Contains("avg:")) doMax = kFALSE; if (oo.Contains("max:")) doAvg = kFALSE; const char *dirn = (isX) ? "RunDataReadx" : "RunDataRead"; TDirectory *d = (TDirectory *) fout->Get(dirn); if (!d) { ::Error("DrawCPU", "could not find directory '%s' ...", dirn); fout->Close(); delete fout; return; } d->cd(); TString hprofn, hmaxn; const char *lx = (isX) ? "_x" : ""; const char *ln = (isNorm) ? "Norm" : "Prof"; Bool_t isIO = kTRUE; if (type && !strcmp(type, "evts")) { hprofn.Form("%s%s_DataRead_QR_Evts", ln, lx); hmaxn.Form("%s%s_DataRead_PS_MaxEvts", ln, lx); isIO = kFALSE; } else { hprofn.Form("%s%s_DataRead_QR_IO", ln, lx); hmaxn.Form("%s%s_DataRead_PS_MaxIO", ln, lx); } Double_t xmin = -1., xmax = -1.; Double_t ami = -1., amx = -1., mmi = -1., mmx = -1.; Int_t kamx = -1, kmmx = -1, nbins = -1; Double_t ymx = -1., ymi = -1.; TProfile *pfav = 0; TGraphErrors *grav = 0; if (doAvg) { if (!(grav = GetGraph(d, hprofn, nbins, xmin, xmax, ami, amx, kamx, pfav))) { ::Error("DrawCPU", "could not find '%s' ...", hprofn.Data()); fout->Close(); delete fout; return; } ymx = amx; ymi = ami; } TProfile *pfmx = 0; TGraphErrors *grmx = 0; if (doMax) { if (!(grmx = GetGraph(d, hmaxn, nbins, xmin, xmax, mmi, mmx, kmmx, pfmx))) { ::Warning("DrawCPU", "could not find '%s': feature added in 5.34/11", hmaxn.Data()); if (!grav) { // Nothing to do if not asked for the average fout->Close(); delete fout; return; } doMax = kFALSE; } if (mmx > ymx) ymx = mmx; if ((ymi > 0 && mmi < ymi) || (ymi < 0.)) ymi = mmi; } TProfile *pf = (doMax) ? pfmx : pfav; Int_t kmx = (doMax) ? kmmx : kamx; // Create the canvas TCanvas *cpu = new TCanvas("dataset", "Rate vs wrks",204,69,1050,502); cpu->Range(-3.106332,0.7490716,28.1362,1.249867); TH1F *hgr = new TH1F("Graph-DataSet"," Data Read speed-up", nbins*4, xmin, xmax); hgr->SetMaximum(ymx + (ymx-ymi)*0.2); hgr->SetMinimum(0); if (isNorm) hgr->SetMaximum(ymx*1.2); hgr->SetDirectory(0); hgr->SetStats(0); hgr->GetXaxis()->SetTitle(pf->GetXaxis()->GetTitle()); hgr->GetXaxis()->CenterTitle(true); hgr->GetXaxis()->SetLabelSize(0.05); hgr->GetXaxis()->SetTitleSize(0.06); hgr->GetXaxis()->SetTitleOffset(0.62); hgr->GetYaxis()->SetLabelSize(0.06); hgr->GetYaxis()->SetTitleSize(0.08); hgr->GetYaxis()->SetTitleOffset(0.52); if (isIO) { hgr->GetYaxis()->SetTitle("Rate (MB/s)"); } else { hgr->GetYaxis()->SetTitle("Rate (events/s)"); } TLegend *leg = 0; if (isNorm) { leg = new TLegend(0.7, 0.8, 0.9, 0.9); } else { leg = new TLegend(0.1, 0.8, 0.3, 0.9); } if (doAvg) { grav->SetFillColor(1); grav->SetLineColor(13); grav->SetMarkerColor(4); grav->SetMarkerStyle(21); grav->SetMarkerSize(1.2); grav->SetHistogram(hgr); if (verbose) grav->Print(); grav->Draw("alp"); leg->AddEntry(grav, "Average", "P"); } if (doMax) { grmx->SetFillColor(1); grmx->SetLineColor(13); grmx->SetMarkerColor(2); grmx->SetMarkerStyle(29); grmx->SetMarkerSize(1.8); grmx->SetHistogram(hgr); if (verbose) grmx->Print(); if (doAvg) { grmx->Draw("lpSAME"); } else { grmx->Draw("alp"); } leg->AddEntry(grmx, "Maximum", "P"); } leg->Draw(); gPad->Update(); Double_t normrate = -1.; if (dofit > 0) { TGraphErrors *gr = (doMax) ? grmx : grav; // Make sure the fitting functions are defined Double_t xmi = 0.9; if (nbins > 5) xmi = 1.5; AssertFittingFun(xmi, nbins + .1); if (dofit == 1) { // Starting point for the parameters and fit fgFio->SetParameter(0, pf->GetBinContent(1)); fgFio->SetParameter(1, pf->GetBinContent(nbins-1)); fgFio->SetParameter(2, pf->GetBinContent(nbins-1)); gr->Fit(fgFio); if (verbose) fgFio->Print(); normrate = fgFio->Derivative(1.); } else if (dofit > 1) { // Starting point for the parameters and fit gFioVn0 = (n0 > 0) ? n0 : (Int_t) (nbins + .1)/2.; gFioVn1 = (n1 > 0) ? n1 : (Int_t) (nbins + .1); fgFioV->SetParameter(0, 20.); fgFioV->SetParameter(1, pf->GetBinContent(1)); fgFioV->SetParameter(2, pf->GetBinContent(1)); fgFioV->SetParameter(3, 4.); fgFioV->SetParameter(4, 1000.); gr->Fit(fgFioV); if (verbose) fgFio->Print(); normrate = fgFioV->Derivative(1.); } } // Notify the cluster performance parameters if (!isNorm) { printf("* ************************************************************ *\n"); printf("* *\r"); printf("* Cluster: %s\n", description.Data()); printf("* Performance measurement from scalability plot: *\n"); printf("* *\r"); if (isIO) { printf("* rate max: %.3f\tMB/s (@ %d workers)\n", ymx, kmx); printf("* *\r"); printf("* per-worker rate: %.3f\tMB/s \n", normrate); } else { printf("* rate max: %.3f\tevts/s (@ %d workers)\n", ymx, kmx); } printf("* ************************************************************ *\n"); } // Close the file fout->Close(); if (grav) fgGraphs->Add(grav); if (grmx) fgGraphs->Add(grmx); } //______________________________________________________________________________ void TProofBench::DrawEfficiency(const char *outfile, const char *opt, Bool_t verbose) { // Draw the efficiency plot. // opt = 'cpu' or 'data' (default the first found) // // Get the TProfile an create the graphs TFile *fout = TFile::Open(outfile, "READ"); if (!fout || (fout && fout->IsZombie())) { ::Error("DrawEfficiency", "could not open file '%s' ...", outfile); return; } // Get description TString description(""); TNamed *nmdesc = (TNamed *) fout->Get("PB_description"); if (nmdesc) description = nmdesc->GetTitle(); // Parse option TString oo(opt), ln("CPU"); const char *dirs[4] = { "RunCPU", "RunCPUx", "RunDataRead", "RunDataReadx"}; const char *labs[4] = { "CPU", "CPU", "DataRead", "DataRead"}; Int_t fst = 0, lst = 3; if (oo == "cpu") { lst = 0; } else if (oo == "cpux") { fst = 1; lst = 1; } else if (oo.BeginsWith("data")) { if (oo.EndsWith("x")) { fst = 3; lst = 3; } else { fst = 2; lst = 2; } } const char *dirn = 0; TDirectory *d = 0; for (Int_t i = fst; i <= lst; i++) { if ((d = (TDirectory *) fout->Get(dirs[i]))) { dirn = dirs[i]; ln = labs[i]; break; } } if (!d && !dirn) { ::Error("DrawEfficiency", "could not find directory ..."); fout->Close(); delete fout; return; } d->cd(); TString hprof; hprof.Form("Prof_%s_CPU_eff", ln.Data()); Double_t xmin = -1., xmax = -1.; Int_t kmx = -1, nbins = -1; Double_t ymx = -1., ymi = -1.; TProfile *pf = 0; TGraphErrors *gr = 0; if (!(gr = GetGraph(d, hprof, nbins, xmin, xmax, ymi, ymx, kmx, pf))) { ::Error("DrawEfficiency", "could not find '%s' ...", hprof.Data()); fout->Close(); delete fout; return; } // Create the canvas TCanvas *cpu = new TCanvas("efficiency", "efficiency vs wrks",204,69,1050,502); cpu->Range(-3.106332,0.7490716,28.1362,1.249867); TH1F *hgr = new TH1F("Graph-Efficiency","CPU effectiveness", nbins*4, xmin, xmax); hgr->SetMaximum(1.2); hgr->SetMinimum(0); hgr->SetDirectory(0); hgr->SetStats(0); hgr->GetXaxis()->SetTitle(pf->GetXaxis()->GetTitle()); hgr->GetXaxis()->CenterTitle(true); hgr->GetXaxis()->SetLabelSize(0.05); hgr->GetXaxis()->SetTitleSize(0.06); hgr->GetXaxis()->SetTitleOffset(0.62); hgr->GetYaxis()->SetLabelSize(0.06); hgr->GetYaxis()->SetTitleSize(0.08); hgr->GetYaxis()->SetTitleOffset(0.52); hgr->GetYaxis()->SetTitle("CPU effectiveness"); gr->SetFillColor(1); gr->SetLineColor(13); gr->SetMarkerColor(4); gr->SetMarkerStyle(21); gr->SetMarkerSize(1.2); gr->SetHistogram(hgr); if (verbose) gr->Print(); gr->Draw("alp"); // Notify the cluster performance parameters printf("* ************************************************************ *\n"); printf("* *\r"); printf("* Cluster: %s\n", description.Data()); printf("* CPU effectiveness measurement: *\n"); printf("* *\r"); printf("* effectiveness max: %.3f (@ %d workers)\n", ymx, kmx); printf("* *\r"); printf("* ************************************************************ *\n"); // Close the file fout->Close(); if (gr) fgGraphs->Add(gr); } //______________________________________________________________________________ Int_t TProofBench::ReleaseCache(const char *dset) { // Release memory cache for dataset 'dset' // Return 0 on success, -1 on error // Do it via the dataset handler if (!fDS) fDS = new TProofBenchDataSet(fProofDS); return fDS ? fDS->ReleaseCache(dset) : -1; } //______________________________________________________________________________ Int_t TProofBench::RemoveDataSet(const char *dset) { // Physically remove the dataset 'dset', i.e. remove the dataset and the files // it describes // Return 0 on success, -1 on error // Do it via the dataset handler if (!fDS) fDS = new TProofBenchDataSet(fProofDS); return fDS ? fDS->RemoveFiles(dset) : -1; } //______________________________________________________________________________ Int_t TProofBench::MakeDataSet(const char *dset, Long64_t nevt, const char *fnroot, Bool_t regenerate) { // Create the largest dataset for the run. // Defaults for // dataset name, filename root // are // "BenchDataSet", "event" // respectively. // These can be changed via dset and fnroot, respectively. // The string 'fnroot' defines the location of the files, interpreted as an URL. // Examples: // fnroot files // 'event' /event__<#>.root // '/mss/event' /mss/event__<#>.root // 'root://srv//mss/event?remote=1' // root://srv//mss/event__<#>?remote=1.root // Default selector is TSelEventGen. Use SetDataGenSel and SetDataGenPar to change it // and to pass the list of PARs defining the alternative selector. // The argument 'nevt' controls the number of events per file (-1 for the default, // which is 30000). // Return 0 on success, -1 on error if (dset && strlen(dset) > 0) fDataSet = dset; // Load the selector, if needed if (!TClass::GetClass(fDataGenSel)) { // Is it the default selector? if (fDataGenSel == kPROOF_BenchSelDataGenDef) { // Load the parfile #ifdef R__HAVE_CONFIG TString par = TString::Format("%s/%s%s.par", ROOTETCDIR, kPROOF_BenchParDir, kPROOF_BenchDataSelPar); #else TString par = TString::Format("$ROOTSYS/etc/%s%s.par", kPROOF_BenchParDir, kPROOF_BenchDataSelPar); #endif Info("MakeDataSet", "uploading '%s' ...", par.Data()); if (fProof->UploadPackage(par) != 0) { Error("MakeDataSet", "problems uploading '%s' - cannot continue", par.Data()); return -1; } Info("MakeDataSet", "enabling '%s' ...", kPROOF_BenchDataSelPar); if (fProof->EnablePackage(kPROOF_BenchDataSelPar) != 0) { Error("MakeDataSet", "problems enabling '%s' - cannot continue", kPROOF_BenchDataSelPar); return -1; } } else { if (fDataGenPar.IsNull()) { Error("MakeDataSet", "you should load the class '%s' before running the benchmark", fDataGenSel.Data()); return -1; } } // Load additional PAR files, if any or required by the alternative selector TString par; Int_t from = 0; while (fDataGenPar.Tokenize(par, from, ",")) { Info("MakeDataSet", "Uploading '%s' ...", par.Data()); if (fProof->UploadPackage(par) != 0) { Error("MakeDataSet", "problems uploading '%s' - cannot continue", par.Data()); return -1; } Info("MakeDataSet", "Enabling '%s' ...", par.Data()); if (fProof->EnablePackage(par) != 0) { Error("MakeDataSet", "problems enabling '%s' - cannot continue", par.Data()); return -1; } } // Check if (!TClass::GetClass(fDataGenSel)) { Error("MakeDataSet", "failed to load '%s'", fDataGenSel.Data()); return -1; } } // For files, 30000 evst each (about 600 MB total) per worker TString fn, fnr("event"); Bool_t remote = kFALSE; if (fnroot && strlen(fnroot) > 0) { TUrl ur(fnroot, kTRUE); if (!strcmp(ur.GetProtocol(), "file") && !gSystem->IsAbsoluteFileName(ur.GetFile())) { fnr = fnroot; } else { fnr = gSystem->BaseName(ur.GetFile()); // We need to set the basedir TString bdir(gSystem->DirName(fnroot)); bdir += "/"; fProof->SetParameter("PROOF_BenchmarkBaseDir", bdir.Data()); // Flag as remote, if so if (strcmp(ur.GetProtocol(), "file")) remote = kTRUE; } } TProofNodes pn(fProof); TMap *filesmap = new TMap; TMap *nodesmap = pn.GetMapOfNodes(); TIter nxnd(nodesmap); TList *wli = 0; TObject *obj = 0; Int_t kf = 1; while ((obj = nxnd()) != 0) { if ((wli = dynamic_cast(nodesmap->GetValue(obj)))) { THashList *fli = new THashList; Int_t nf = wli->GetSize() * fNFilesWrk; TSlaveInfo *wi = (TSlaveInfo *) wli->First(); while (nf--) { fn.Form("%s-%s-%d.root", fnr.Data(), wi->GetName(), kf++); // Add to the node list for generation fli->Add(new TObjString(fn)); } filesmap->Add(new TObjString(obj->GetName()), fli); } } filesmap->Print(); // Prepare for file generation ... add map in the input list filesmap->SetName("PROOF_FilesToProcess"); fProof->AddInput(filesmap); // Set parameters for processing TString oldpack; if (TProof::GetParameter(fProof->GetInputList(), "PROOF_Packetizer", oldpack) != 0) oldpack = ""; fProof->SetParameter("PROOF_Packetizer", "TPacketizerFile"); Int_t oldnotass = -1; if (TProof::GetParameter(fProof->GetInputList(), "PROOF_ProcessNotAssigned", oldnotass) != 0) oldnotass = -1; fProof->SetParameter("PROOF_ProcessNotAssigned", (Int_t)0); // Process Long64_t ne = (nevt > 0) ? nevt : 30000; fProof->SetParameter("PROOF_BenchmarkNEvents", ne); fProof->SetParameter("PROOF_BenchmarkRegenerate", Int_t(regenerate)); fProof->Process(fDataGenSel, (Long64_t) 1); fProof->DeleteParameters("PROOF_BenchmarkNEvents"); fProof->DeleteParameters("PROOF_BenchmarkRegenerate"); fProof->DeleteParameters("PROOF_BenchmarkBaseDir"); // Restore parameters if (!oldpack.IsNull()) fProof->SetParameter("PROOF_Packetizer", oldpack); else fProof->DeleteParameters("PROOF_Packetizer"); if (oldnotass != -1) fProof->SetParameter("PROOF_ProcessNotAssigned", oldnotass); else fProof->DeleteParameters("PROOF_ProcessNotAssigned"); // Cleanup if (fProof->GetInputList()) fProof->GetInputList()->Remove(filesmap); filesmap->SetOwner(kTRUE); delete filesmap; // The dataset to be registered in the end with proper port TFileCollection *fc = new TFileCollection("dum", "dum"); if (fProof->GetOutputList()) { fProof->GetOutputList()->Print(); TIter nxout(fProof->GetOutputList()); while ((obj = nxout())) { TList *fli = dynamic_cast(obj); if (fli && TString(fli->GetName()).BeginsWith("PROOF_FilesGenerated_")) { TIter nxfg(fli); TFileInfo *fi = 0; while ((fi = (TFileInfo *) nxfg())) fc->Add(fi); fli->SetOwner(kFALSE); } } // Register the new dataset, overwriting any existing dataset wth the same name // trusting the existing information fc->Update(); if (fc->GetNFiles() > 0) { if (remote) fc->SetBit(TFileCollection::kRemoteCollection); if (!(fProof->RegisterDataSet(fDataSet, fc, "OT"))) Warning("MakeDataSet", "problems registering '%s'", dset); } else { Warning("MakeDataSet", "dataset '%s' is empty!", dset); } } else { Warning("MakeDataSet", "PROOF output list is empty!"); } SafeDelete(fc); // Get updated information fc = fProof->GetDataSet(fDataSet); if (fc) { fc->Print("F"); } else { Warning("MakeDataSet", "dataset '%s' was not generated!", fDataSet.Data()); } SafeDelete(fc); // Done return 0; } //______________________________________________________________________________ Int_t TProofBench::CopyDataSet(const char *dset, const char *dsetdst, const char *destdir) { // Copy the files of dataset 'dset' to 'destdir' and create a new dataset named 'dsetdst' // decribing them. // Return 0 on success, -1 on error // Make some checks if (!fProof) { Error("CopyDataSet", "no PROOF found - cannot continue"); return -1; } if (!dset || (dset && !fProof->ExistsDataSet(dset))) { Error("CopyDataSet", "dataset '%s' does not exist", dset); return -1; } if (!dsetdst || (dsetdst && fProof->ExistsDataSet(dsetdst))) { if (isatty(0) != 0 && isatty(1) != 0) { Printf("Target dataset '%s' exists already:" " do you want to remove it first?", dsetdst); const char *a = Getline("[Y,n] "); Printf("a: %s", a); if (a[0] == 'Y' || a[0] == 'y' || a[0] == '\n') { Info("CopyDataSet", "removing dataset '%s' ...", dsetdst); RemoveDataSet(dsetdst); } else { return -1; } } else { Error("CopyDataSet", "destination dataset '%s' does already exist: remove it first", dsetdst); return -1; } } // The TFileCollection object for the new dataset TFileCollection *fc = fProof->GetDataSet(dset); if (!fc) { Error("CopyDataSet", "problems retrieving TFileCollection for dataset '%s'", dset); return -1; } TFileCollection *fcn = new TFileCollection(dsetdst, ""); TString fn; TFileInfo *fi = 0; TIter nxfi(fc->GetList()); while ((fi = (TFileInfo *) nxfi())) { fn.Form("%s/%s", destdir, gSystem->BaseName(fi->GetCurrentUrl()->GetFile())); Info("CopyDataSet", "adding info for file '%s'", fn.Data()); fcn->Add(new TFileInfo(fn)); } delete fc; // Do it via the dataset handler if (!fDS) fDS = new TProofBenchDataSet(fProofDS); if (fDS->CopyFiles(dset, destdir) != 0) { Error("CopyDataSet", "problems copying files of dataset '%s' to dest dir '%s'", dset, destdir); delete fcn; return -1; } // Register the new dataset, overwriting any existing dataset wth the same name // trusting the existing information Int_t rc = 0; if (!(fProof->RegisterDataSet(dsetdst, fcn, "OT"))) { Error("CopyDataSet", "problems registering and verifying '%s'", dsetdst); rc = -1; } delete fcn; // Done return rc; } //______________________________________________________________________________ void TProofBench::SetProofDS(TProof *pds) { // Set the PROOF instance to be used for dataset operations, like releasing // cache ... // Use SetProofDS(0) to reset and using the default PROOF if (pds && !pds->IsValid()) { Error("SetProofDS", "trying to set an invalid PROOF instance"); return; } fProofDS = pds ? pds : fProof; if (fProofDS) { SafeDelete(fDS); fDS = new TProofBenchDataSet(fProofDS); } // Done return; }