Project

General

Profile

fclmodule.cxx

Kyle Knoepfel, 04/17/2017 08:48 AM

 
1
//----------------------------------------------------------------------
2
//
3
// Name: fclmodule.cxx
4
//
5
// Purpose: A python extension module that wraps the c++ fcl
6
//          api.  The main purpose of this module is to provide the
7
//          capability to read and parse a fcl file and return the
8
//          resulting parameter set in the form of a python dictionary.
9
//
10
//          This module compiles to a shared library (fclmodule.so).  
11
//          When this shared library is on the python path, it can be
12
//          imported as a python module (import fcl).
13
//
14
// Created: 4-Apr-2017  H. Greenlee
15
//
16
// FCL module interface.
17
//
18
// Module members.
19
//
20
// 1.  Function make_pset(fclfile)
21
//
22
// The argument is the name of a fcl file (can be anywhere on $FHICL_FILE_PATH).
23
// The function reads the fcl file and returns the expanded parameter set
24
// in the form of a python dictionary.
25
//
26
// Example:
27
//
28
// #! /usr/bin/env python
29
// import fcl
30
// pset = fcl.make_pset('myfile.fcl')
31
//
32
//
33
//----------------------------------------------------------------------
34

    
35
#include "Python.h"
36
#include <iostream>
37
#include <iomanip>
38
#include <sstream>
39
#include "cetlib/search_path.h"
40
#include "fhiclcpp/ParameterSet.h"
41
#include "fhiclcpp/ParameterSetRegistry.h"
42
#include "fhiclcpp/make_ParameterSet.h"
43

    
44
// Forward declarations.
45

    
46
PyObject* convert_any(const boost::any& any);
47

    
48
PyObject* convert_atom(const std::string& atom)
49
//
50
// Purpose: Convert a string representing an atomic value to a python object.
51
//          The type of the returned object is one of: bool, int, float, or 
52
//          string.  The type is chosen based on the string content.  In
53
//          principle, fhicl also support type complex.  We don't handle 
54
//          complex in this function (complex atomic values will be returned
55
//          as strings).
56
//
57
// Arguments: atom - A string reprsenting an atomic value.
58
//
59
// Return value: Python object pointer.
60
//
61
{
62
  // Return value.
63

    
64
  PyObject* pyval = 0;
65

    
66
  // Get lower case version of argument string.
67

    
68
  std::string lcatom(atom);
69
  std::transform(lcatom.begin(), lcatom.end(), lcatom.begin(),
70
                 [](unsigned char c){return std::tolower(c);});
71

    
72
  // Check for boolean.
73

    
74
  if(lcatom == std::string("true")) {
75
    pyval = Py_True;
76
    Py_INCREF(pyval);
77
  }
78
  else if(lcatom == std::string("false")) {
79
    pyval = Py_False;
80
    Py_INCREF(pyval);
81
  }
82

    
83
  // Check for quoted string.
84

    
85
  size_t n = atom.size();
86
  if(pyval == 0 && n >= 2 && atom[0] == '"' && atom[n-1] == '"') {
87
    std::string s = atom.substr(1, n-2);
88
    pyval = PyString_FromString(s.c_str());
89
  }  
90

    
91
  // Check for int.
92

    
93
  if(pyval == 0) {
94
    std::istringstream iss(atom);
95
    long i;
96
    iss >> std::noskipws >> i;
97
    if(iss.eof() and !iss.fail())
98
      pyval = PyInt_FromLong(i);
99
  }
100

    
101
  // Check for float.
102

    
103
 if(pyval == 0) {
104
    std::istringstream iss(atom);
105
    double x;
106
    iss >> std::noskipws >> x;
107
    if(iss.eof() and !iss.fail())
108
      pyval = PyFloat_FromDouble(x);
109
  }
110
  
111
  // Last resort store a copy of the original string (unquoted string).
112

    
113
  if(pyval == 0)
114
    pyval = PyString_FromString(atom.c_str());
115

    
116
  // Done.
117

    
118
  return pyval;
119
}
120

    
121
PyObject* convert_pset(const fhicl::ParameterSet& pset)
122
//
123
// Purpose: Convert a parameter set to a python dictionary.
124
//
125
// Arguments: pset - Parameter set.
126
//
127
// Return value: Python dictionary pointer.
128
//
129
{
130
  // Make an empty python dictionary that will be our result.
131

    
132
  PyObject* pydict = PyDict_New();
133

    
134
  // Pry open the parameter set.
135

    
136
  const std::map<std::string, boost::any>& anymap = 
137
    reinterpret_cast<const std::map<std::string, boost::any>&>(pset);
138

    
139
  // Loop over items in parameter set.
140

    
141
  for(const auto& entry : anymap) {
142
    const std::string& key = entry.first;
143
    const boost::any& any = entry.second;
144
    PyObject* pyval = convert_any(any);
145

    
146
    // Done converting key.
147

    
148
    if(pyval != 0) {
149

    
150
      // Conversion was successful.  Insert value into dictionary.
151

    
152
      PyDict_SetItemString(pydict, key.c_str(), pyval);
153
      Py_DECREF(pyval);
154
    }
155
    else {
156

    
157
      // Abort.
158

    
159
      Py_DECREF(pydict);
160
      pydict = 0;
161
      break;
162
    }
163
  }
164

    
165
  // Done.
166

    
167
  return pydict;
168
}
169

    
170
PyObject* convert_seq(const std::vector<boost::any>& seq)
171
//
172
// Purpose: Convert a sequence to a python list.
173
//
174
// Arguments: seq - Sequence.
175
//
176
// Return value: Python list pointer.
177
//
178
{
179
  // Make a python list that will be our return value.
180

    
181
  PyObject* pylist = PyList_New(seq.size());
182
  for(unsigned int i=0; i<seq.size(); ++i) {
183

    
184
    // Convert element.
185

    
186
    PyObject* pyele = convert_any(seq[i]);
187
    if(pyele != 0) {
188

    
189
      // Element conversion was successful.  Insert element into list.
190

    
191
      PyList_SetItem(pylist, i, pyele);
192
    }
193
    else {
194

    
195
      // Abort.
196

    
197
      Py_DECREF(pylist);
198
      pylist = 0;
199
    }
200
  }
201

    
202
  // Done.
203

    
204
  return pylist;
205
}
206

    
207
PyObject* convert_any(const boost::any& any)
208
//
209
// Purpose: Convert a boost::any to a python object.
210
//
211
// Arguments: any - Boost::any object.
212
//
213
// Return value: Python object pointer.
214
//
215
{
216
  // Return value.
217

    
218
  PyObject* pyval = 0;
219

    
220
  if(any.type() == typeid(std::string)) {
221

    
222
    // Atomic types.
223

    
224
    const std::string& atom = boost::any_cast<const std::string&>(any);
225
    pyval = convert_atom(atom);
226
  }
227
  else if(any.type() == typeid(std::vector<boost::any>)) {
228

    
229
    // Sequences.
230

    
231
    const std::vector<boost::any>& anyvec = boost::any_cast<const std::vector<boost::any>&>(any);
232
    pyval = convert_seq(anyvec);
233
  }
234
  else if(any.type() == typeid(fhicl::ParameterSetID)) {
235

    
236
    // Parameter sets.
237

    
238
    const fhicl::ParameterSetID& id = boost::any_cast<const fhicl::ParameterSetID&>(any);
239
    const fhicl::ParameterSet& pset = fhicl::ParameterSetRegistry::get(id);
240
    pyval = convert_pset(pset);
241
  }
242
  else {
243

    
244
    // Unknown type.
245
    // Shouldn't happen.
246

    
247
    std::string msg = std::string("Failed to convert object of type ") + any.type().name();
248
    PyErr_SetString(PyExc_ValueError, msg.c_str());
249
  }
250

    
251
  // Done.
252

    
253
  return pyval;
254
}
255

    
256
static std::string format(PyObject* obj, unsigned int pos, unsigned int indent, unsigned int maxlen)
257
//
258
// Purpose: Convert a python object to a prettified string.  The resulting string
259
//          is suppsed to be valid python code.
260
//
261
// Arguments: obj    - Object to be formatted.
262
//            pos    - Current character position (number of characters printed
263
//                     since the last newline).
264
//            indent - Indentation level (spaces) for multiline formatting.
265
//            maxlen - Maximum line length before breaking.
266
//
267
// Returns: c++ string.
268
//
269
// Usage:
270
//
271
// This function is designed to call itself recursively in order to descend
272
// into structured objects like dictionaries and sequences.
273
//
274
// Dictionaries and sequences may be printed in either single-line or multiline
275
// format, depending on the complexity of the contained objects, and the indent
276
// and maxlen parameters.
277
//
278
{
279
  // Result string stream.
280

    
281
  std::ostringstream ss;
282

    
283
  if(PyString_Check(obj)) {
284

    
285
    // String objects, add single quotes, but don't do any other formatting.
286

    
287
    ss << "'" << PyString_AsString(obj) << "'";
288
  }
289

    
290
  else if(PyDict_Check(obj)) {
291

    
292
    // Always print dictionary objects in multiline format, one key per line.
293

    
294
    // Get list of keys.  Keys are assumed to be strings.
295

    
296
    PyObject* keys = PyDict_Keys(obj);
297

    
298
    // Make a first pass over the list of keys to determine the maximum length key.
299

    
300
    int n = PyList_Size(keys);
301
    int keymaxlen = 0;
302
    for(int i=0; i<n; ++i) {
303
      PyObject* key = PyList_GetItem(keys, i);
304
      int keylen = PyString_Size(key);
305
      if(keylen > keymaxlen)
306
        keymaxlen = keylen;
307
    }
308

    
309
    // Second pass, loop over keys and values and convert them to strings.
310

    
311
    char sep = '{';
312
    for(int i=0; i<n; ++i) {
313
      PyObject* key = PyList_GetItem(keys, i);
314
      PyObject* value = PyDict_GetItem(obj, key);
315
      const char* ks = PyString_AsString(key);
316
      std::string ksquote = std::string("'") + std::string(ks) + std::string("'");
317
      ss << sep << '\n'
318
         << std::setw(indent+2) << ""
319
         << std::setw(keymaxlen+2) << std::left << ksquote << " : "
320
         << format(value, indent + keymaxlen + 7, indent+2, maxlen);
321
      sep = ',';
322
    }
323
    ss << '\n' << std::setw(indent+1) << std::right << '}';
324

    
325
    Py_DECREF(keys);
326

    
327
  }
328

    
329
  else if(PyList_Check(obj) || PyTuple_Check(obj)) {
330

    
331
    // Sequence printing handled here.
332
    // Break lines only when position exceeds maxlen.
333

    
334
    char open_seq = '(';
335
    char close_seq = ')';
336
    if(PyList_Check(obj)) {
337
      open_seq = '[';
338
      close_seq = ']';
339
    }
340

    
341
    // Loop over elements of this sequence.
342

    
343
    std::string sep(1, open_seq);
344
    int n = PyList_Size(obj);
345
    unsigned int break_indent = pos+1;
346
    for(int i=0; i<n; ++i) {
347
      ss << sep;
348
      pos += sep.size();
349
      PyObject* ele = PySequence_GetItem(obj, i);
350

    
351
      // Get the formatted string representation of this object.
352

    
353
      std::string f = format(ele, pos, break_indent, maxlen);
354

    
355
      // Get the number of characters before the first newline.
356

    
357
      std::string::size_type fs = f.size();
358
      std::string::size_type n1 = std::min(f.find('\n'), fs);
359

    
360
      // Decide if we want to break the line before printing this element.
361
      // Never break at the first element of a sequence.
362
      // Force a break (except at first element) if this is a structured element.
363
      // If we do break here, reformat this element with the updated position.
364

    
365
      bool force_break = PyList_Check(ele) || PyTuple_Check(ele) || PyDict_Check(ele);
366
      if(i > 0 && (force_break || pos + n1 > maxlen)) {
367
        ss << '\n' << std::setw(break_indent) << "";
368
        pos = break_indent;
369
        f = format(ele, pos, break_indent, maxlen);
370
      }
371

    
372
      // Print this element
373

    
374
      ss << f;
375

    
376
      // Update the current character position, taking into account 
377
      // whether the string we just printed contains a newline.
378

    
379
      std::string::size_type n2 = f.find_last_of('\n');
380
      if(n2 >= fs)
381
        pos += fs;
382
      else
383
        pos = fs - n2 - 1;
384

    
385
      sep = std::string(", ");
386
      Py_DECREF(ele);
387
    }
388

    
389
    // Close sequence.
390

    
391
    ss << close_seq;
392
  }
393

    
394
  else {
395

    
396
    // Last resort, use python's string representation.
397

    
398
    PyObject* pystr = PyObject_Str(obj);
399
    ss << PyString_AsString(pystr);
400
  }
401

    
402
  // Done.
403

    
404
  return ss.str();
405
}
406

    
407
static PyObject* make_pset(PyObject* self, PyObject *args)
408
//
409
// Purpose: Public module function to read fcl file and return a python dictionary.
410
//
411
// Arguments: self - Not used, because this is not a member function.
412
//            args - Argument tuple.  A single string representing the
413
//                   name of a fcl file.
414
// Returned value: A python dictionary.
415
//
416
{
417
  // Extract argument as string.
418

    
419
  const char* fclname;
420
  if(!PyArg_ParseTuple(args, "s", &fclname))
421
    return 0;
422
  std::string fclstr(fclname);
423

    
424
  // Make parameter set.
425

    
426
  PyObject* result = 0;
427
  fhicl::ParameterSet pset;
428
  std::string pathvar("FHICL_FILE_PATH");
429
  cet::filepath_lookup maker(pathvar);
430
  try {
431
    fhicl::make_ParameterSet(fclstr, maker, pset);
432
    result = convert_pset(pset);
433
  }
434
  catch(cet::exception& e) {
435
    PyErr_SetString(PyExc_IOError, e.what());
436
    result = 0;
437
  }
438

    
439
  // Done.
440

    
441
  return result;
442
}
443

    
444
static PyObject* pretty(PyObject* self, PyObject *args)
445
//
446
// Purpose: Public module function to convert a python fcl dictionary to a
447
//          prettified string.
448
//
449
// Arguments: self - Not used, because this is not a member function.
450
//            args - Argument tuple.  This function expects a single 
451
//                   python object of any type, but typically a dictionary.
452
//
453
// Returned value: A python string or none.
454
//
455
{
456
  // Result.
457

    
458
  PyObject* result = 0;
459

    
460
  // Extract argument.
461

    
462
  int n = PySequence_Length(args);
463
  if(n == 0) {
464

    
465
    // No arguments, return none.
466

    
467
    result = Py_None;
468
    Py_INCREF(result);
469
  }
470
  else {
471

    
472
    // Otherwise, extract the first element.
473

    
474
    PyObject* obj = PySequence_GetItem(args, 0);
475
    std::string s = format(obj, 0, 0, 80);
476
    result = PyString_FromString(s.c_str());
477
  }
478

    
479
  // Done.
480

    
481
  return result;
482
}
483

    
484
// Module method table.
485

    
486
static struct PyMethodDef fclmodule_methods[] = {
487
     {"make_pset", make_pset, METH_VARARGS},
488
     {"pretty", pretty, METH_VARARGS},
489
     {0, 0}
490
};
491

    
492
// Initialization function.
493

    
494
extern "C" {
495
  void initfcl()
496
  {
497
    Py_InitModule("fcl", fclmodule_methods);
498
  }
499
}