added logic for limits.computation.memory.buffer

include/control/parameters.hpp

@@ -612,11 +612,13 @@ private:
     friend class boost::serialization::access;	
 	template<class Archive> void serialize(Archive & ar, const unsigned int version)
     {
+        ar & buffer;
 		ar & signal;
     }
 	/////////////////// 
 
 public:
+    size_t buffer;
     size_t signal;
 };
 

src/control/parameters.cpp

@@ -486,7 +486,8 @@ void Params::read_xml(std::string filename) {
     limits.signal.chunksize = 10000;
         
     limits.computation.threads = 1;
-    limits.computation.memory.signal = 200*1024*1024; // 100MB 
+    limits.computation.memory.signal = 200*1024*1024; // 200MB 
+    limits.computation.memory.buffer = 200*1024*1024; // 200MB 
 
     limits.services.signal.memory.server = 100*1024*1024; // 100MB
     limits.services.signal.memory.client = 10*1024*1024; // 10MB
@@ -523,6 +524,9 @@ void Params::read_xml(std::string filename) {
             	if (xmli.exists("//limits/computation/memory/signal")) {
     	            limits.computation.memory.signal = xmli.get_value<size_t>("//limits/computation/memory/signal");
     	        }
+            	if (xmli.exists("//limits/computation/memory/buffer")) {
+    	            limits.computation.memory.buffer = xmli.get_value<size_t>("//limits/computation/memory/buffer");
+    	        }
 	        }
 	    }
 	    

src/scatter_devices/all_vectors_scatter_device.cpp

@@ -187,6 +187,18 @@ void AllVectorsScatterDevice::compute() {
     // special case: 1 core, no exchange required
     if (NMBLOCK==1) {
         size_t NTHREADS = worker_threads.size();
+        
+        size_t bufferbytesize = Params::Inst()->limits.computation.memory.buffer;
+        size_t bufferbytesize_requested = NF*NTHREADS*sizeof(fftw_complex);
+        bufferbytesize_requested+=2*NF*sizeof(fftw_complex); // account for alignpad here...
+        if (bufferbytesize_requested>bufferbytesize) {
+            if (allcomm_.rank()==0) {
+                Err::Inst()->write("Computation buffer too small.");
+                Err::Inst()->write(string("limits.computation.memory.buffer=")+boost::lexical_cast<string>(bufferbytesize));
+                Err::Inst()->write(string("requested: ")+boost::lexical_cast<string>(bufferbytesize_requested));            
+            }
+            throw;
+        }
         at_ = (fftw_complex*) fftw_malloc(NF*NTHREADS*sizeof(fftw_complex));
         memset(at_,0,NF*NTHREADS*sizeof(fftw_complex));
 
@@ -210,8 +222,22 @@ void AllVectorsScatterDevice::compute() {
             current_subvector_+=NTHREADS;
             p_monitor_->update(allcomm_.rank(),progress());
         }
-                       
+        fftw_free(at_);        
+        
     } else {
+        size_t bufferbytesize = Params::Inst()->limits.computation.memory.buffer;
+        size_t bufferbytesize_requested=0;
+        bufferbytesize_requested+=NMAXF*NMBLOCK*sizeof(fftw_complex); // initial buffer
+        bufferbytesize_requested+=NMAXF*NMBLOCK*sizeof(fftw_complex); // clone during exchange   
+        bufferbytesize_requested+=2*NF*sizeof(fftw_complex); // account for alignpad here...
+        if (bufferbytesize_requested>bufferbytesize) {
+            if (allcomm_.rank()==0) {
+                Err::Inst()->write("Computation buffer too small.");
+                Err::Inst()->write(string("limits.computation.memory.buffer=")+boost::lexical_cast<string>(bufferbytesize));
+                Err::Inst()->write(string("requested: ")+boost::lexical_cast<string>(bufferbytesize_requested));            
+            }
+            throw;
+        }
         at_ = (fftw_complex*) fftw_malloc(NMAXF*NMBLOCK*sizeof(fftw_complex));
         memset(at_,0,NMAXF*NMBLOCK*sizeof(fftw_complex));
 

src/scatter_devices/self_vectors_scatter_device.cpp

@@ -119,7 +119,22 @@ void SelfVectorsScatterDevice::compute() {
 	memset(atfinal_,0,NF*sizeof(fftw_complex));
     
     size_t NTHREADS = worker_threads.size();
+    
+    // total memory requirements during computation:
+    // atfinal = NF
+    // at_ = NTHREADS*2*NF
+    
     // double allocate for zero padding
+    size_t bufferbytesize = Params::Inst()->limits.computation.memory.buffer;
+    size_t bufferbytesize_requested = 2*NF*NTHREADS*sizeof(fftw_complex);
+    if (bufferbytesize_requested>bufferbytesize) {
+        if (allcomm_.rank()==0) {
+            Err::Inst()->write("Computation buffer too small.");
+            Err::Inst()->write(string("limits.computation.memory.buffer=")+boost::lexical_cast<string>(bufferbytesize));
+            Err::Inst()->write(string("requested: ")+boost::lexical_cast<string>(bufferbytesize_requested));            
+        }
+        throw;
+    }
     at_ = (fftw_complex*) fftw_malloc(2*NF*NTHREADS*sizeof(fftw_complex));
     memset(at_,0,2*NF*NTHREADS*sizeof(fftw_complex));
 
@@ -149,6 +164,8 @@ void SelfVectorsScatterDevice::compute() {
     }
     timer.stop("sd:c:block");
     
+    fftw_free(at_);
+    
     timer.start("sd:c:wait");
     partitioncomm_.barrier();
     timer.stop("sd:c:wait");