Project

General

Profile

Early success in running two RCE BoardReaders with RunControl and DAQInterface

07-Aug-2014, John Freeman, Tim Nicholls, Matt Thorpe, Kurt Biery

Steps for running RC and DAQI (from John)

On the gateway node:

cd /data/lbnedaq/scratch/DAQInterface
. .bashrc
. fireup

And, after checking that lbnecontrol and daqinterface aren't running:

lbnecontrol &
daqinterface -n daqint -r 5570 -c localhost -H localhost &

lbnecmd init daq
lbnecmd start daq
lbnecmd stop daq
lbnecmd terminate daq

And to see what state daqinterface is in:

lbnecmd check

FHiCL files

Aggregator_lbnedaq2_1.fcl

services: {
  scheduler: {
    fileMode: NOMERGE
  }
  user: {
    NetMonTransportServiceInterface: {
      service_provider: NetMonTransportService
      max_fragment_size_words: 2097152
    }
  }
  Timing: { summaryOnly: true }
  #SimpleMemoryCheck: { }
}

daq: {
  max_fragment_size_words: 2097152
  aggregator: {
    mpi_buffer_count: 8
    first_event_builder_rank: 2
    event_builder_count: 2
    expected_events_per_bunch: 1
    print_event_store_stats: true
    event_queue_depth: 20
    event_queue_wait_time: 5
    onmon_event_prescale: 1
    xmlrpc_client_list: ";http://lbnedaq2:5205/RPC2,3;http://lbnedaq2:5206/RPC2,3;http://lbnedaq2:5235/RPC2,4;http://lbnedaq2:5236/RPC2,4;http://lbnedaq2:5265/RPC2,5;http://lbnedaq2:5266/RPC2,5" 
    file_size_MB: 8000.0
    file_duration: 0
    file_event_count: 0
  }
}

source: {
  module_type: NetMonInput
}
outputs: {
  normalOutput: {
    module_type: RootOutput
    fileName: "/data/lbnedaq/data/lbne_r%06r_sr%02s_%to.root" 
  }
}
physics: {
  analyzers: {

  }

  producers: {
  }

  #a1: [ app, wf ]

  my_output_modules: [ normalOutput ]
}
process_name: DAQAG

Aggregator_lbnedaq2_2.fcl

services: {
  scheduler: {
    fileMode: NOMERGE
  }
  user: {
    NetMonTransportServiceInterface: {
      service_provider: NetMonTransportService
      max_fragment_size_words: 2097152
    }
  }
  Timing: { summaryOnly: true }
  #SimpleMemoryCheck: { }
}

daq: {
  max_fragment_size_words: 2097152
  aggregator: {
    mpi_buffer_count: 8
    first_event_builder_rank: 2
    event_builder_count: 2
    expected_events_per_bunch: 1
    print_event_store_stats: true
    event_queue_depth: 2
    event_queue_wait_time: 1
    onmon_event_prescale: 1
    xmlrpc_client_list: ";http://lbnedaq2:5205/RPC2,3;http://lbnedaq2:5206/RPC2,3;http://lbnedaq2:5235/RPC2,4;http://lbnedaq2:5236/RPC2,4;http://lbnedaq2:5265/RPC2,5;http://lbnedaq2:5266/RPC2,5" 
    file_size_MB: 8000.0
    file_duration: 0
    file_event_count: 0
  }
}

source: {
  module_type: NetMonInput
}
outputs: {
  #normalOutput: {
  #  module_type: RootOutput
  #  fileName: "/data/lbnedaq/data/lbne_r%06r_sr%02s_%to.root" 
  #}
}
physics: {
  analyzers: {

  }

  producers: {
  }

  #a1: [ app, wf ]

  #my_output_modules: [ normalOutput ]
}
process_name: DAQAG  

BoardReader_TPC_lbnedaq2_1.fcl

daq: {
  max_fragment_size_words: 262144
  fragment_receiver: {
    mpi_buffer_count: 16
    mpi_sync_interval: 50
    first_event_builder_rank: 2
    event_builder_count: 2

        generator: TpcRceReceiver
    fragment_type: TPC
    fragment_id: 0
    board_id: 0
    sleep_on_stop_us: 500000 

 number_of_microslices_per_millislice: 100

 rce_client_host_addr: lbnedaq3 
 rce_client_host_port: 9991
 rce_client_timeout_usecs: 500000

 rce_data_dest_host: '10.0.1.2'
 rce_data_dest_port: 8991  
 rce_data_num_millislices: 0    # 0=Run until stopped
 rce_data_num_microslices: 100
 rce_data_frag_rate: 1000.0
 rce_data_adc_mode: 4
 rce_data_adc_mean: 1000.0
 rce_data_adc_sigma: 100.0

 receive_port: 8991

 receiver_debug_level: 1
 receiver_tick_period_usecs: 10000

 reporting_interval_fragments: 100

 raw_buffer_size: 1000000
 raw_buffer_precommit: 10 
 use_fragments_as_raw_buffer: true

    }
}

BoardReader_TPC_lbnedaq2_2.fcl

daq: {
  max_fragment_size_words: 262144
  fragment_receiver: {
    mpi_buffer_count: 16
    mpi_sync_interval: 50
    first_event_builder_rank: 2
    event_builder_count: 2

        generator: TpcRceReceiver
    fragment_type: TPC
    fragment_id: 1
    board_id: 1
    sleep_on_stop_us: 500000 

 number_of_microslices_per_millislice: 100

 rce_client_host_addr: lbnedaq3 
 rce_client_host_port: 9992
 rce_client_timeout_usecs: 500000

 rce_data_dest_host: '10.0.1.2'
 rce_data_dest_port: 8992  
 rce_data_num_millislices: 0    # 0=Run until stopped
 rce_data_num_microslices: 100
 rce_data_frag_rate: 1000.0
 rce_data_adc_mode: 4
 rce_data_adc_mean: 1000.0
 rce_data_adc_sigma: 100.0

 receive_port: 8992

 receiver_debug_level: 1
 receiver_tick_period_usecs: 10000

 reporting_interval_fragments: 100

 raw_buffer_size: 1000000
 raw_buffer_precommit: 10 
 use_fragments_as_raw_buffer: true

    }
}

EventBuilder_lbnedaq2_1.fcl

services: {
  scheduler: {
    fileMode: NOMERGE
  }
  user: {
    NetMonTransportServiceInterface: {
      service_provider: NetMonTransportService
      first_data_receiver_rank: 4
      mpi_buffer_count: 8
      max_fragment_size_words: 2097152
      data_receiver_count: 1 # 2
      #broadcast_sends: true
    }
  }
  Timing: { summaryOnly: true }
  #SimpleMemoryCheck: { }
}

daq: {
  max_fragment_size_words: 2097152
  event_builder: {
    mpi_buffer_count: 16
    first_fragment_receiver_rank: 0
    fragment_receiver_count: 2
    expected_fragments_per_event: 2
    use_art: true
    print_event_store_stats: true
    verbose: false
  }
} 

outputs: {
  netMonOutput: {
    module_type: NetMonOutput
  }
  #normalOutput: {
  #  module_type: RootOutput
  #  fileName: "/data/lbnedaq/data/lbne_eb02_r%06r_sr%02s_%to.root" 
  #  compressionLevel: 0
  #}
}

physics: {
  analyzers: {

  }

  producers: {
  }

  #a1: [ app, wf ]

  my_output_modules: [ netMonOutput ]
  #my_output_modules: [ normalOutput ]
}
source: {
  module_type: RawInput
  waiting_time: 900
  resume_after_timeout: true
  fragment_type_map: [[1, "missed"], [2, "TPC"], [3, "PHOTON"], [4, "TRIGGER"], [5, "TOY1"], [6, "TOY2"]]
}
process_name: DAQ

EventBuilder_lbnedaq2_2.fcl

services: {
  scheduler: {
    fileMode: NOMERGE
  }
  user: {
    NetMonTransportServiceInterface: {
      service_provider: NetMonTransportService
      first_data_receiver_rank: 4
      mpi_buffer_count: 8
      max_fragment_size_words: 2097152
      data_receiver_count: 1 # 2
      #broadcast_sends: true
    }
  }
  Timing: { summaryOnly: true }
  #SimpleMemoryCheck: { }
}

daq: {
  max_fragment_size_words: 2097152
  event_builder: {
    mpi_buffer_count: 16
    first_fragment_receiver_rank: 0
    fragment_receiver_count: 2
    expected_fragments_per_event: 2
    use_art: true
    print_event_store_stats: true
    verbose: false
  }
} 

outputs: {
  netMonOutput: {
    module_type: NetMonOutput
  }
  #normalOutput: {
  #  module_type: RootOutput
  #  fileName: "/data/lbnedaq/data/lbne_eb02_r%06r_sr%02s_%to.root" 
  #  compressionLevel: 0
  #}
}

physics: {
  analyzers: {

  }

  producers: {
  }

  #a1: [ app, wf ]

  my_output_modules: [ netMonOutput ]
  #my_output_modules: [ normalOutput ]
}
source: {
  module_type: RawInput
  waiting_time: 900
  resume_after_timeout: true
  fragment_type_map: [[1, "missed"], [2, "TPC"], [3, "PHOTON"], [4, "TRIGGER"], [5, "TOY1"], [6, "TOY2"]]
}
process_name: DAQ

Relevant data file:

ROOT file written during this run, containing two TPC fragments per event, is on 35T system at: /data/lbnedaq/data/lbne_r000200_sr01_20140807T150347.root