\documentclass[english,serif,mathserif,xcolor=pdftex,dvipsnames,table]{beamer}
\usepackage{lsci}

\title[ARClib]{%
  Introduction to ARClib
  with~Python, II
}
\author[R. Murri]{%
  \textbf{Riccardo Murri} \\
  Grid Computing Competence Center, \\
  Organisch-Chemisches Institut, \\
  University of Zurich
}
\date{Nov.~24,~2011}

% for typesetting "C++", see
% http://stackoverflow.com/questions/2724760/how-to-write-c-in-latex
\usepackage{relsize}
\newcommand\ifmonospace{%
  \ifdim\fontdimen3\font=0pt%
}
\newcommand\Cpp{%
  \ifmonospace%
  C++%
  \else%
  C\kern-.0667em\raise.20ex\hbox{\relsize{-2}{++}}%
  \fi%
  \spacefactor1000%
}


\begin{document}

% title frame
\maketitle


\section{Introduction}

\begin{frame}
  \frametitle{Today's class}
  
  An introduction to using ARClib from Python.

  \+
  Or how to re-implement the
  \texttt{arcsub}/\texttt{arcstat}/\texttt{arcget} 
  commands using Python and good will.

  \+
  {\small These slides are available for download from: 
    \url{http://www.gc3.uzh.ch/teaching/lsci2011/lab09.pdf}}
\end{frame}


\section{arcsub}

\begin{frame}
  \frametitle{Job submission}
  Job submission in ARC is a multi-step process.

  \begin{enumerate}
  \item Create an xRSL description of the job.
  \item Get the list of \emph{queues}, i.e., potential submission targets.
  \item Select the actual submission targets \emph{(brokering)}.
  \item Perform the actual submission.
  \end{enumerate}

  \+ 
  Each of these steps has a dedicated ARClib call.
\end{frame}


\begin{frame}[fragile]
  \frametitle{Creating xRSL objects}
  First, it is necessary to create an ARClib \lstinline|Xrsl| object
  corresponding to the textual xRSL description of a job.

  \begin{describe}{\lstinline|arclib.Xrsl(text)|}
    Return the ARClib \lstinline|Xrsl| object
    corresponding to the xRSL string \lstinline|text|.
  \end{describe}

  \begin{python}
>>> with open('env.xrsl') as stream:
...     xrsl_text = stream.read()
>>> xrsl = arclib.Xrsl(xrsl_text)
>>> dir(xrsl)
['AddRelation', 'AddSimpleRelation', 'AddXrsl', 
 'Eval', 'GetAllRelations', 'GetRelation', 
 'IsRelation', 'Print', 'RemoveRelation', 
 'SplitMulti', 'SplitOrRelation', 'Validate', !\ldots!]
  \end{python}
\end{frame}


\begin{frame}[fragile]
  \frametitle{Listing queues}
  \small
  A queue is a subset of a cluster with certain defined properties,
  e.g., maximum allowed run time of jobs.

  \+
  So you do not submit to clusters, but to \emph{queues}.

  \begin{describe}{\lstinline|arclib.GetQueueInfo(cluster_urls)|}
    Return a sequence of \lstinline|arclib.Queue| objects, each
    corresponding to a queue.  The \lstinline|cluster_urls| argument
    is a sequence in the format returned by \lstinline|arclib.GetClusterResources|.
  \end{describe}

\end{frame}

\begin{frame}[fragile]
  \frametitle{What's in a \texttt{arclib.Queue} object?}
  It's a Python equivalent of the LDAP entry.
  \begin{lstlisting}[basicstyle=\scriptsize\ttfamily]
>>> q = qs[0] # pick a queue from the list
>>> dir(q)
[!\ldots!, 'architecture', 'benchmarks', 'cluster', 
 'comment', 'cpu_freq', 'default_cpu_time', 
 'default_wall_time', 'grid_queued', 'grid_running', 
 'homogeneity', 'local_queued', 'max_cpu_time', 
 'max_queuable', 'max_running', 'max_total_cpu_time', 
 'max_user_run', 'max_wall_time', 'mds_validfrom', 
 'mds_validto', 'middlewares', 'min_cpu_time', 
 'min_wall_time', 'name', 'node_cpu', 'node_memory', 
 'operating_systems', 'prelrms_queued', 'queued', 
 'running', 'runtime_environments', 'scheduling_policy', 
 'status', 'this', 'total_cpus', 'users']
>>> q.node_memory
2000
>>> q.max_cpu_time # undefined numeric arguments get -1
-1
>>> q.node_cpu # undefined string arguments get empty string
''
  \end{lstlisting}

  {\small
    Reference: \url{http://www.nordugrid.org/documents/arc_infosys.pdf}}
\end{frame}


\begin{frame}[fragile]
  \frametitle{Select submission targets (brokering)}
  \begin{describe}{\lstinline|arclib.ContructTargets(queues, xrsl)|}
    Prepare for brokering, by converting sequence of \emph{queues}
    into a sequence of \emph{targets}.
  \end{describe}

  \begin{describe}{\lstinline|arclib.PerformStandardBrokering(targets)|}
    Sort the \lstinline|targets| sequence according to preference,
    using ``standard'' broker (configured in the
    \lstinline|~/.arc/client.conf| file), and at the same time remove
    those targets that are incompatible with the given xRSL. Return
    sorted sequence.
  \end{describe}
\end{frame}

\begin{frame}[fragile]
  \frametitle{Brokering example}
  \begin{lstlisting}[basicstyle=\footnotesize\ttfamily]
>>> # filter incompatible queues
>>> ts = arclib.ConstructTargets(qs, xrsl)

>>> # sort them according to rank
>>> ts = arclib.PerformStandardBrokering(ts)

>>> # print out results
>>> for t in ts: print t.cluster, '@', t.cpu_freq
... 
Cluster: bacchus.switch.ch  SWITCH SMSCG TEST @ -1.0
Cluster: ocikbnor.uzh.ch  OCI Grid Cluster @ 2992.49804688
Cluster: smscg.inf.usi.ch  USI-ICS Cluster @ -1.0
Cluster: smscg.epfl.ch  SMSCG - EPFL @ 3325.03808594
Cluster: gordias.unige.ch  Gordias at hepia @ 2800.0
Cluster: ce.lhep.unibe.ch  Bern ATLAS T3 @ 2400.0
Cluster: hera.wsl.ch  WSL Grid Cluster @ 2712.50390625
Cluster: idgc3grid01.uzh.ch  GC3 Grid Cluster @ 800.0
Cluster: nordugrid.unibe.ch  Bern UBELIX T3 Cluster @ 2200.0
Cluster: globus.vital-it.ch  SMSCG - Vital-IT @ -1.0
  \end{lstlisting}
\end{frame}


\begin{frame}
  \frametitle{Perform job submission}
  \begin{describe}{\lstinline|arclib.SubmitJob(xrsl, targets)|}
    Try to subit a job to the given sequence of targets; return the
    job ID (a string) if submission succeeds, else raise
    \lstinline|arclib.JobSubmissionError|.

    \+
    The submission operation takes an \lstinline|Xrsl| object,
    and a sequence of targets (as returned by the
    \lstinline|arclib.PerformStandardBrokering| call.

    \+ All targets will be tried in order until one succeeds.

    \+ 
    If none succeeds, an \lstinline|arclib.JobSubmissionError|
    exception will be raised.

    \+
    \emph{Note:} the return value is a Python string representing a
    URL, not an \lstinline|arclib.URL| object.
  \end{describe}
\end{frame}


\begin{frame}[fragile]
  \begin{exercise}
    Write a \lstinline|submit_job| function that takes a string
    argument (the xRSL description of a job) and returns the job ID of
    the submitted job (if successful).
  \end{exercise}

  \+
  \begin{exercise}
    Write a program \lstinline|jobsubmit.py| that takes any number of
    ``\texttt{.xrsl}'' files on the command line, tries to submit all
    of them, and for each file outputs a line with the name of the
    file, the submission status (\texttt{OK} or \texttt{FAIL}) and the
    job ID.  In case of failure, also print the exception message.
\begin{semiverbatim}\footnotesize
\$ \textbf{./jobsubmit.py env.xrsl largemem.xrsl}
env.xrsl: OK: gsiftp://idgc3grid01.uzh.ch/jobs/123456789123456789
largemem.xrsl: FAIL: No targets available for job-submission
\end{semiverbatim}
  \end{exercise}
\end{frame}


\section{arcstat}

\begin{frame}
  \frametitle{Contacting the GIIS}
  ARC uses a central information ``index'' , accessible over the
  \href{http://en.wikipedia.org/wiki/Ldap}{LDAP protocol}.

  \+ 
  To gather information about any entity in the Grid (cluster, job,
  user), you contact this information index (called the GIIS), which
  transparently redirects you to the LDAP server (called GRIS) of the
  site that keeps the information.

  \+
  The GIIS of the SMSCG infrastructure can be contacted at the URL
  {\small \url{ldap://giis.smscg.ch:2135/o=grid/mds-vo-name=switzerland}}
\end{frame}


\begin{frame}[fragile]
  \frametitle{ARClib does URLs}
  ARClib provides its own facility for (Grid) URL handling.

  \+
  Create an ARClib URL object:
  \begin{lstlisting}[basicstyle=\footnotesize\ttfamily]
>>> giis_url = arclib.URL('ldap://giis.smscg.ch:2135'
...                       '/o=grid/mds-vo-name=switzerland')
  \end{lstlisting}

  \+
  The logical parts of the URL are available through accessor method
  calls:
  \begin{lstlisting}[basicstyle=\footnotesize\ttfamily]
>>> giis_url.CanonicalURL()
'ldap://giis.smscg.ch:2135/o=grid/mds-vo-name=switzerland'
>>> giis_url.Host()
'giis.smscg.ch'
>>> giis_url.Port()
2135
>>> giis_url.BaseDN()
'mds-vo-name=switzerland, o=grid'
  \end{lstlisting}
\end{frame}


\begin{frame}[fragile]
  \frametitle{Listing resources}
  Listing available computing resources is simple:
  \begin{python}
>>> clusters = arclib.GetClusterResources(giis_url)
  \end{python}

  \+
  The returned value is a tuple of \lstinline|arclib.URL| objects:
  \begin{python}
>>> clusters
(<arclib.URL; ... >, <arclib.URL; proxy of <Swig Object of type 'URL *' ...> >, ...
>>> type(clusters)
<type 'tuple'>
>>> type(clusters[0])
<class 'arclib.URL'>
  \end{python}

  \+
  \begin{exercise}
    Write a function \lstinline|list_clusters(url)| that returns the
    list of Internet host names of all the clusters registered at the
    given GIIS URL.  Assume the \lstinline|url| argument is a string.
  \end{exercise}
\end{frame}

\begin{frame}[fragile]
  \frametitle{Listing jobs on a cluster}
  \small
  The following functions in ARClib allow you to retrieve information
  on jobs running on a remote cluster.

  \begin{describe}{\lstinline|arclib.GetClusterJobs(cluster)|}
    Return a sequence of \lstinline|arclib.Job| objects, one for each
    job that is currently managed by the remote cluster.  The sole
    argument \lstinline|cluster| is a \lstinline|arclib.URL| instance
    pointing to a cluster (as returned by
    \lstinline|arclib.GetClusterResources|). 
  \end{describe}

  \begin{describe}{\lstinline|arclib.GetClusterJobs(cluster_list)|}
    Same as above, but the argument is a \lstinline|list| of cluster URLs.
  \end{describe}

  \begin{describe}{\lstinline|arclib.GetAllJobs(cluster_list)|}
    Same as above, but restricted to one's own jobs.
  \end{describe}
\end{frame}

\begin{frame}[fragile,fragile]
  \frametitle{What's a Job like, anyway?}
\begin{lstlisting}[basicstyle=\scriptsize\ttfamily]
>>> jobs = arclib.GetClusterJobs(cluster)
>>> job = jobs[0]
\end{lstlisting}
  As usual, we can use \lstinline|dir()| to inspect live objects:
  \begin{lstlisting}[basicstyle=\scriptsize\ttfamily]
>>> dir(job)
[!\ldots!, 'cluster', 'comment', 'completion_time', 'cpu_count',
'erase_time', 'errors', 'execution_nodes', 'exitcode', 'gmlog', 
'id', 'job_name', 'mds_validfrom', 'mds_validto', 'owner', 
'proxy_expire_time', 'queue', 'queue_rank', 'requested_cpu_time', 
'requested_wall_time', 'rerunable', 'runtime_environments', 
'sstderr', 'sstdin', 'sstdout', 'status', 'submission_time', 
'submission_ui', 'this', 'used_cpu_time', 'used_memory', 'used_wall_time']
>>> job.id
'gsiftp://smscg.inf.usi.ch:2811/jobs/143741321446490878298264'
>>> job.cluster
'smscg.inf.usi.ch'
>>> job.owner
'/DC=com/DC=quovadisglobal/DC=grid/DC=switch/DC=users/C=CH/O=SWITCH/CN=Placi Flury'
>>> job.status
'INLRMS:Q'
  \end{lstlisting}
\end{frame}

% \begin{frame}[fragile]
%   \frametitle{Interlude: emulating \texttt{dict}'s}

%   Any instance \lstinline|x| of a class that defines a 
%   \lstinline|__getitem__(self, key)| method can use dictionary-like
%   lookup syntax \lstinline|x[key]|:
%   \begin{python}
% >>> class MyDict(UserDict.DictMixin):
% ...   def __getitem__(self, key):
% ...     return 42
% >>> x = MyDict()
% >>> x['whatever']
% 42
%   \end{python}

%   \+
%   Inheriting from the standard library class
%   \lstinline|UserDict.DictMixin| is needed to define other
%   dictionary-like methods on top of \lstinline|__getitem__|.
  
%   \begin{seealso}
%     {\scriptsize \url{http://docs.python.org/library/userdict.html#module-UserDict}}
%     \hbox{\scriptsize\url{http://docs.python.org/reference/datamodel.html#emulating-container-types}}
%   \end{seealso}
% \end{frame}


% \begin{frame}
%   \begin{exercise}
%     Implement a dictionary-like object \lstinline|JobsByCluster|, mapping
%     cluster names to \lstinline|arclib.Job| objects.
%     \begin{itemize}
%     \item The constructor takes two arguments: a \lstinline|GridAuth|
%       instance, and an optional GIIS URL (a string).  The GIIS URL
%       defaults to the SMSCG one.
%     \item  The \lstinline|Job| objects associated with a cluster name
%       are the jobs registered with the remote cluster owned by the
%       ``Subject DN'' of the \lstinline|GridAuth| passed to the
%       constructor. 
%     \end{itemize}
%   \end{exercise}
% \end{frame}


\begin{frame}[fragile]
  \begin{exercise}
    Implement a program \texttt{joblist.py}:
    \begin{itemize}
    \item If called without command-line arguments, prints a list of
      job names and corresponding \emph{status} and \emph{job ID}s running at all available
      clusters; the cluster name is used as a heading before the job
      IDs assigned to that cluster.  Example:
      \begin{lstlisting}[language=sh]
        $ !\textbf{./joblist.py}!
        === a.cluster.name ===
        name1: status jobID1
        name2: status jobID2
        ....
        === another.cluster.here ===
        name: status jobID
        ....
      \end{lstlisting}
    \item If called with arguments, those argument must be cluster
      names: only the jobs assigned to those clusters must be listed. 
    \end{itemize}
  \end{exercise}
\end{frame}


\begin{frame}[fragile]
  \begin{exercise}\small
    Write a \lstinline|Job| class according to the following specification:
    \begin{itemize}
      \item The \lstinline|Job| constructor takes three parameters:
        \lstinline|jobid|, \lstinline|cluster|, \lstinline|giis_url|;
        all three are strings, the last two are optional.
        \begin{description}
        \item[jobid] is the ARC job URL;
        \item[cluster] is the Internet host name of the cluster;
        \item[giis\_url] is the URL of the GIIS to contact,
          defaults to the SMSCG one.
        \end{description}
      \item A \lstinline|Job| instance shall have attributes
        \lstinline|owner|, \lstinline|status|, \lstinline|exitcode|,
        \lstinline|errors|, having the same value of corresponding
        attributes in the \lstinline|arclib.Job| attribute, plus a
        \lstinline|cluster| attribute which is the internet host name
        of the cluster where the job was submitted.
      \item The \lstinline|Job| class shall have an
        \lstinline|update()| method, which sets up-to-date values for
        the \lstinline|status|, \lstinline|exitcode|,
        \lstinline|errors|.
    \end{itemize}
  \end{exercise}
\end{frame}


\begin{frame}[fragile]
  \frametitle{Timing execution}
  Python's standard \lstinline|time| module provides facilities for
  measuring elapsed time.
  
  \begin{python}
>>> import time
>>> t0 = time.time() # current time, in seconds
>>> time.sleep(5)    # simulate doing something
>>> t1 = time.time()
>>> elapsed = t1 - t0
>>> print elapsed
5.00510191917
  \end{python}
\end{frame}

\begin{frame}
  \frametitle{Homework 1}
  Using the Python timing facilities, find out which is the fastest
  alternative in each pair:
  \begin{enumerate}
  \item Calling \texttt{GetClusterJobs()} with no parameters vs.\
    running a loop \lstinline|for cluster: GetClusterJobs(cluster)|
    and accumulating results.
  \item Calling \texttt{GetClusterJobs()} and then filtering out
    non-own jobs vs.\ calling \texttt{GetAllJobs()}
  \end{enumerate}

  \+
  Your script should make at least three runs of each function, and
  pick the shortest time of the three, then compare it with the
  corresponding timing for the other function.

  \+ 
  Make a table of the performance (absolute and ratio).
\end{frame}

\section{arcget}

\begin{frame}
  \frametitle{Downloading job output}
  The \texttt{gsiftp://} prefix on the job IDs indicates that jobs are
  accessible through \href{http://www.ogf.org/documents/GFD.20.pdf}{GridFTP}.  
  In particular, you can retrieve job output files using an FTP-like interface.

  \+
  ARClib provides functions to perform GridFTP listings and transfers.
  
  \begin{describe}{\lstinline|arclib.JobFTPControl()|}
    Initialize the object interface used to interact with GridFTP
    servers. 
  \end{describe}

  \begin{describe}{\lstinline|job_ftp_control.DownloadDirectory(url, destination)|}
    Download the directory pointed to by \lstinline|url| (a string)
    into the local path \lstinline|destination|, \emph{recursing into
      subdirectories} at any depth.
  \end{describe}
\end{frame}


\begin{frame}
  \begin{exercise}
    Write a \lstinline|jobdownload.py| program, that takes a job ID
    and a destination directory path on the command line, and
    downloads the entire output of the job to the specified directory.
    Intercept exceptions and print an appropriate error message.
  \end{exercise}
\end{frame}


\begin{frame}[fragile]
  \frametitle{More GridFTP operations, I}
  There's one entry point for all GridFTP-related operations: the
  class \lstinline|FTPControl|.

  \+
  The class constructor is called with no parameters:
\begin{python}
>>> ftp = arclib.FTPControl()    
\end{python}

  \+
  \lstinline|FTPControl| objects expose methods for uploading,
  downloading remote files and directories, and listing remote
  directories.
\end{frame}


\begin{frame}
  \frametitle{More GridFTP operations, II}
  \begin{describe}{\lstinline|ftp_control.Download(url, local_file)|}
    Download the whole contents of the file stored at \lstinline|url|
    into local path \lstinline|local_file|.

    If \lstinline|url| is a directory or unavailable, an exception
    \lstinline|arclib.FTPControlError| is raised.
  \end{describe}

  \begin{describe}{\lstinline|ftp_control.Download(url, offset, size, local_file)|}
    As above, but only download the \lstinline|size| bytes starting at
    byte \lstinline|offset|.
  \end{describe}

  \begin{describe}{\lstinline|ftp_control.Upload(local_file, url)|}
    Upload the whole contents of the file stored at local path
    \lstinline|local_file| to \lstinline|url|.

    If \lstinline|url| is a directory or unavailable, an exception
    \lstinline|arclib.FTPControlError| is raised.
  \end{describe}
\end{frame}


\begin{frame}
  \begin{exercise}
    \emph{1.} Write a \texttt{gridftp.py} program, that takes three arguments
    from the command-line: \lstinline|action|, \lstinline|src| and
    \lstinline|dest|. 

    \+
    The \lstinline|action| argument can be either \lstinline|'get'| or
    \lstinline|'put'|.

    \+
    If \lstinline|action| is \lstinline|get|, then \lstinline|src| is
    a URL and \lstinline|dest| is a local path; \texttt{gridftp.py}
    shall download the contents of \lstinline|src| onto
    \lstinline|dest|.

    \+
    If \lstinline|action| is \lstinline|put|, then \lstinline|src| is
    a local path and \lstinline|dest| is a URL; \texttt{gridftp.py}
    shall upload the contents of \lstinline|src| onto
    \lstinline|dest|.

    \+
    \emph{2.} What happens if you try to download a URL onto an
    \emph{existing} local file?
  \end{exercise}
\end{frame}


\begin{frame}
  \frametitle{More GridFTP operations, III}
  \begin{describe}{\lstinline|ftp_control.ListDir(url)|}
    List the contents of directory pointed to by \lstinline|url| (a
    string). Return a sequence of \lstinline|arclib.FileInfo| objects,
    which expose attributes \lstinline|filename| (a string expressing
    the path relative to the FTP server root), \lstinline|isdir| (a
    boolean), and \lstinline|size| (for files, the size in bytes; for
    directories, always 0)
  \end{describe}

  \begin{describe}{\lstinline|ftp_control.RecursiveListDir(url)|}
    Same as above, but descend directories recursively.
  \end{describe}
\end{frame}


\begin{frame}
  \begin{exercise}
    \emph{1.} Expand the \texttt{gridftp.py} program from the last
    exercise with a \lstinline|list| action that lists the contents of
    a remote directory (specified by URL).

    \+
    \emph{2.} Modify \texttt{gridftp.py} so that, if a directory URL is given to
    the \lstinline|put| action, then it appends the local file name to
    it to form a file URL.  (You can use \lstinline|os.path.basename|
    and \lstinline|os.path.join| to manipulate path strings.)

    \+ 
    \emph{3.}  Add an action \lstinline|find| to \texttt{gridftp.py}:
    it takes a directory URL and a second argument
    \lstinline|pattern|; if \lstinline|pattern| is given, then the
    remote directory should be scanned recursively, and the URLs of
    the files matching the given pattern should be printed.  (You can
    use Python's standard module
    \href{http://docs.python.org/library/fnmatch.html}{\lstinline|fnmatch|}
    to check if a filename matches a pattern.)
  \end{exercise}
\end{frame}


\begin{frame}
  \frametitle{Homework 2}
  Re-architect the \lstinline|Job| class from  exercise~5 so
  that:
  \begin{enumerate}[{\itshape (a)}]
  \item No more than one \lstinline|GetClusterJobs| or
    \lstinline|GetClusterResources| call (one per type) is issued in
    30 seconds.
  \item The information returned by \lstinline|GetClusterJobs| is
    shared among all the \lstinline|Job| instances referring to jobs
    assigned to the same cluster.
  \end{enumerate}
\end{frame}

%\section{arcsub}

% each cluster has 1..N queues
% use GetQueueInfo() to retrieve list



\end{document}

%%% Local Variables: 
%%% mode: latex
%%% TeX-master: t
%%% End: