Project

General

Profile

Modules POMS refurbish (Initial plan)

The intention of this work is to make POMS easy to maintain, more modular and have the possibility to test the modules separated.

Description

Nearly all the methods in poms_service should look like:

def method_name(arguments...):
vars = self.blah.method_name(cherrypy.request.db, arguments...)
template = self.jinja_env.get_template('method_name.html')
vars.update( {
'current_experimenter': cherrypy.session.get('experimenter'),
'pomspath':self.path,
'help_page':"MethodNameHelp"
})
return template.render(**vars)
with their guts factored out into other classes, which should have unit tests.

Current Status of POMS

Out of POMS Class

def error_response(): (new)
def popen_read_with_timeout(cmd, totaltime = 30): (new)

leave in/split up

def init(self):
def headers(self):
def sign_out(self):
def index(self): (in the proposal it is in downtimes, should it be in CalendarPOMS.py?)
def es(self): (new)

Security (Pending - original proposal)

def can_report_data(self):
def can_db_admin(self):

> > def jump_to_job(self, jobsub_job_id, **kwargs ): (previous in utils, should it be in jobPOMS.py?)

CalendarPOMS (completed - pending to test)

def calendar_json(self, dbhandle,start, end, timezone, _):
def calendar(self, dbhandle):
def add_event(self, dbhandle,title, start, end):

def edit_event [not in the module]

service_downtimes(self, dbhandle):
update_service(self, dbhandle, log_handle, name, parent, status, host_site, total, failed, description):
service_status(self, dbhandle, under = 'All'):

> > def service_status_hier(self, under = 'All', depth = 0) (where should be this one?, should it maybe include a new template?, it is working in the dashboard)

> > def new_task_for_campaign(self, campaign_name, command_executed, experimenter_name, dataset_name = None): (should it be on task or in campigns?) -

> > def make_admin_map(self): (should it be inside DBAdminPOMS.py ?) (previous in raw tables)

DBAdminPOMS.py (Complete - pending to test - you call it experiments in the first schema )

def raw_tables(self): [not in the module] (previous in Raw tables)

def user_edit(self, dbhandle, *args, **kwargs):
def experiment_members(self, dbhandle, *args, **kwargs):
def experiment_edit(self, dbhandle):
def experiment_authorize(self, dbhandle, loghandle, *args, **kwargs):

>>>>>
Should be a new module call* RawtablesPOMS*

def list_generic(self, classname): (previous in Raw tables)
def edit_screen_generic(self, classname, id = None):
def update_generic( self, classname, *args, **kwargs):
def update_for( self, classname, eclass, primkey, *args , **kwargs):
def edit_screen_for( self, classname, eclass, update_call, primkey, primval, valmap):
def make_list_for(self,eclass,primkey):

>>>>>

JobsPOMS.py

def active_jobs(self, dbhandle):

def report_declared_files(self, flist): [not in the module]

def output_pending_jobs(self,dbhandle):
def update_job(self, dbhandle, loghandle, rpstatus, task_id = None, jobsub_job_id = 'unknown', **kwargs):

TaskPOMS.py

def create_task(self, dbhandle, experiment, taskdef, params, input_dataset, output_dataset, creator, waitingfor = None ): (previous in fancy_tables)
def wrapup_tasks(self, dbhandle, samhandle):
def compute_status(self, task):
def show_task_jobs(self, task_id, tmax = None, tmin = None, tdays = 1 ): previous campaing task monitoring

> > def triage_job(self, job_id, tmin = None, tmax = None, tdays = None, force_reload = False): (previous in triage module)

>>>>>>>>>>>>>>>>>>>>
def handle_dates(self,tmin, tmax, tdays, baseurl): (previous in util)
def show_campaigns(self,experiment = None, tmin = None, tmax = None, tdays = 1, active = True): (previous campaign_task_monitoring)
def campaign_info(self, campaign_id, tmin = None, tmax = None, tdays = None): (previous campaign_task_monitoring)
def list_task_logged_files(self, task_id): (file_status)
def campaign_task_files(self,campaign_id, tmin = None, tmax = None, tdays = 1): (file_status)
def campaign_time_bars(self, campaign_id, tmin = None, tmax = None, tdays = 1): (previous campaign_task_monitoring) [ class fakerow: def init(self, *kwargs): ]
def task_min_job(self, task_id): *(previous in util)

def job_file_list(self, job_id,force_reload = False): (file_status)
def job_file_contents(self, job_id, task_id, file, tmin = None, tmax = None, tdays = None): (file_status)
test_job_counts(self, task_id = None, campaign_id = None): (previous in util)
def format_job_counts(self, task_id = None, campaign_id = None, tmin = None, tmax = None, tdays = 7, range_string = None): (file_status)
def job_counts(self, task_id = None, campaign_id = None, tmin = None, tmax = None, tdays = None): (previous in triage module)
def job_table(self, tmin = None, tmax = None, tdays = 1, task_id = None, campaign_id = None , experiment = None, sift=False, campaign_name=None, name=None,campaign_def_id=None, vo_role=None, input_dataset=None, output_dataset=None, task_status=None, project=None, jobsub_job_id=None, node_name=None, cpu_type=None, host_site=None, job_status=None, user_exe_exit_code=None, output_files_declared=None, campaign_checkbox=None, task_checkbox=None, job_checkbox=None, ignore_me = None, keyword=None, dataset = None, eff_d = None): (previous in triage module)
def jobs_by_exitcode(self, tmin = None, tmax = None, tdays = 1 ): (previous in triage module)
def failed_jobs_by_whatever(self, tmin = None, tmax = None, tdays = 1 , f = [], go = None): (previous in triage module)
def get_task_id_for(self, campaign, user = None, experiment = None, command_executed = "", input_dataset = "", parent_task_id=None): (JobPOMS or TaskPOMS)
def register_poms_campaign(self, experiment, campaign_name, version, user = None, campaign_definition = None, dataset = "", role = "Analysis", params = []): (JobPOMS or TaskPOMS or CampaingsPOMS)
def quick_search(self, search_term): (previous in util)

json_project_summary_for_task(self, task_id): (file_status)
project_summary_for_task(self, task_id): (file_status)
project_summary_for_tasks(self, task_list): (file_status)
get_inflight(self, campaign_id=None, task_id=None): (file_status)
show_dimension_files(self, experiment, dims): (file_status)
def inflight_files(self, campaign_id=None, task_id=None): (file_status)
def campaign_sheet(self, campaign_id, tmin = None, tmax = None , tdays = 7): (file_status or CampaignPOMS)

def kill_jobs(self, campaign_id=None, task_id=None, job_id=None, confirm=None): (JobsPOMS)

def get_dataset_for(self, camp): (CampaignPOMS)
def set_job_launches(self, hold): (JobsPOMS )
def launch_dependents_if_needed(self, task_id): (JobsPOMS )
def get_recovery_list_for_campaign_def(self, campaign_def): * [new method, Campaign or Jobs]*
launch_recovery_if_needed(self, task_id): (CampaignPOMS or JobsPOMS)
launch_jobs(self, campaign_id, dataset_override = None, parent_task_id = None): (CampaignPOMS or JobsPOMS)

def link_tags(self, campaign_id, tag_name, experiment): (previous tags)
def delete_campaigns_tags(self, campaign_id, tag_id, experiment): (previous tags)
def search_tags(self, q): (previous tags)
def auto_complete_tags_search(self, experiment, q): (previous tags)

def jobs_eff_histo(self, campaign_id, tmax = None, tmin = None, tdays = 1 ): (previous campaign_task_monitoring)
def list_launch_file(self, campaign_id, fname ): (CampaignPOMS or JobsPOMS)
def schedule_launch(self, campaign_id ): (CampaignPOMS or JobsPOMS or TaskPOMS)
def update_launch_schedule(self, campaign_id, dowlist = None, domlist = None, monthly = None, month = None, hourlist = None, submit = None , minlist = None, delete = None): (CampaignPOMS or JobsPOMS or TaskPOMS)*

def snapshot_parts(self, t, campaign_id): (CampaignPOMS or JobsPOMS or TaskPOMS or launchPOMS)

def mark_campaign_active(self, campaign_id, is_active): (CampaignPOMS) [new method]
def make_stale_campaigns_inactive(self): (CampaignPOMS) [new method]

def actual_pending_files(self, count_or_list, task_id = None, campaign_id = None, tmin = None, tmax= None, tdays = 1): (file_status)
def get_efficiency(self, campaign_list, tmin, tmax): (CampaignPOMS) [new method]
def get_pending_for_campaigns(self, campaign_list, tmin, tmax): (CampaignPOMS) [new method]
def get_pending_for_task_lists(self, task_list_list): (CampaignPOMS) [new method]


Description of the modules

CalendarPOMS.py

Methods:
  • def calendar_json(self, dbhandle,start, end, timezone, _):
  • def calendar(self, dbhandle):
  • def add_event(self, dbhandle,title, start, end):
  • service_downtimes(self, dbhandle):
  • update_service(self, dbhandle, log_handle, name, parent, status, host_site, total, failed, description):
  • service_status(self, dbhandle, under = 'All'):

DBAdminPOMS.py

Methods:
  • def user_edit(self, dbhandle, *args, **kwargs):
  • def experiment_members(self, dbhandle, *args, **kwargs):
  • def experiment_edit(self, dbhandle):
  • def experiment_authorize(self, dbhandle, loghandle, *args, **kwargs):

JobsPOMS.py

Methods:
  • def active_jobs(self, dbhandle):
  • def output_pending_jobs(self,dbhandle):
  • def update_job(self, dbhandle, loghandle, rpstatus, task_id = None, jobsub_job_id = 'unknown', **kwargs):
  • def test_job_counts(self, task_id = None, campaign_id = None):

TaskPOMS.py

Methods:
  • def create_task(self, dbhandle, experiment, taskdef, params, input_dataset, output_dataset, creator, waitingfor = None ):
  • def wrapup_tasks(self, dbhandle, samhandle):
  • def show_task_jobs(self, task_id, tmax = None, tmin = None, tdays = 1 ):
  • def compute_status(self, task):
  • def task_min_job(self, dbhandle, task_id):

CampaignsPOMS.py

Methods:
  • def launch_template_edit(self, dbhandle, loghandle, seshandle, *args, **kwargs):
  • def campaign_definition_edit(self, dbhandle, loghandle, seshandle, *args, **kwargs):
  • def campaign_edit(self, dbhandle, loghandle, sesshandle, *args, **kwargs):
  • def campaign_edit_query(self, dbhandle, *args, **kwargs):

Methods under construction

UtilsPOMS.py

Methods:

def handle_dates(self,tmin, tmax, tdays, baseurl):
def quick_search(self, dbhandle, redirect, search_term):
def jump_to_job(self, dbhandle, redirect ,jobsub_job_id, **kwargs ):

FilesPOMS.py

Methods:

def list_task_logged_files(self, dbhandle, task_id):
def campaign_task_files(self, dbhandle, loghandle, samhandle, campaign_id, tmin = None, tmax = None, tdays = 1):
def job_file_list(self, dbhandle, jobhandle, job_id, force_reload = False):
def job_file_contents(self, dbhandle, loghandle, jobhandle, job_id, task_id, file, tmin = None, tmax = None, tdays = None):
def format_job_counts(self, task_id = None, campaign_id = None, tmin = None, tmax = None, tdays = 7, range_string = None):
def get_inflight(self, dbhandle, campaign_id=None, task_id=None):
def inflight_files(self, dbhandle, status_response, campaign_id=None, task_id=None):
def show_dimension_files(self, samhandle, experiment, dims):
def campaign_sheet(self, dbhandle, loghandle, campaign_id, tmin = None, tmax = None , tdays = 7):
def actual_pending_files(self, dbhandle, loghandle, count_or_list, task_id = None, campaign_id = None, tmin = None, tmax= None, tdays = 1):

TriagePOMS.py

Methods:

def job_counts(self, task_id = None, campaign_id = None, tmin = None, tmax = None, tdays = None):
def triage_job(self, dbhandle, job_id, tmin = None, tmax = None, tdays = None, force_reload = False):
def job_table(self, dbhandle, tmin = None, tmax = None, tdays = 1, task_id = None, campaign_id = None , experiment = None, sift=False, campaign_name=None, name=None,campaign_def_id=None, vo_role=None, input_dataset=None, output_dataset=None, task_status=None, project=None, jobsub_job_id=None, node_name=None, cpu_type=None, host_site=None, job_status=None, user_exe_exit_code=None, output_files_declared=None, campaign_checkbox=None, task_checkbox=None, job_checkbox=None, ignore_me = None, keyword=None, dataset = None, eff_d = None):
def failed_jobs_by_whatever(self, dbhandle, loghandle, tmin = None, tmax = None, tdays = 1 , f = [], go = None):

def jobs_by_exitcode #should be in the poms_service code

TagsPOMS.py

Methods:

def link_tags(self, dbhandle, campaign_id, tag_name, experiment):
def delete_campaigns_tags(self, dbhandle, ses_get, campaign_id, tag_id, experiment):
def search_tags(self, q):
def auto_complete_tags_search(self, dbhandle, experiment, q):

TablesPOMS

def list_generic(self, classname): (previous in Raw tables)
def edit_screen_generic(self, classname, id = None):
def update_generic( self, classname, *args, **kwargs):
def update_for( self, classname, eclass, primkey, *args , **kwargs):
def edit_screen_for( self, classname, eclass, update_call, primkey, primval, valmap):
def make_list_for(self,eclass,primkey):