Project

General

Profile

Jupyter with CDO

Added by Ilias Moysidis about 1 year ago

I am trying to run a python script in a jupyter notbook that uses some cdo functions. This is how I loaded the libraries that I use:

import sys
# !conda install --yes --prefix {sys.prefix} cdo
# !{sys.executable} -m pip install cdo
import os
from zipfile import ZipFile
from cdo import *

For some reason I am getting an error when I try to use the mergetime function of the CDO library. However, I can run the same script in my computer without any problems. This is the error that I am receiving:

---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
Cell In[12], line 15
     11 ofile = os.path.join('datasets', variable, scenario, model_name)
     13 # decompress_file(input=ifile, output=ofile)
     14 # rename_cordex(ofile)
---> 15 create_decade(ofile)
     16 print(create_decade_chunks(ofile))

Cell In[11], line 60, in create_decade(directory)
     57 second_part_of_decade_name = chunk[-1].rsplit('-', 1)[-1]
     58 ofile = '-'.join([first_part_of_decade_name, second_part_of_decade_name])
---> 60 Cdo().mergetime(options='-z zip', input=ifile, output=ofile)
     61 [os.remove(file) for file in chunk]

File ~/anaconda3/lib/python3.10/site-packages/cdo.py:190, in Cdo.__init__(self, cdo, returnNoneOnError, forceOutput, env, debug, tempdir, logging, logFile, cmd, options, silent)
    187 self._cmd = cmd
    188 self._options = options
--> 190 self.operators         = self.__getOperators()
    191 self.noOutputOperators = [op for op in self.operators.keys() if 0 == self.operators[op]]
    192 self.returnNoneOnError = returnNoneOnError

File ~/anaconda3/lib/python3.10/site-packages/cdo.py:283, in Cdo.__getOperators(self)
    280 def __getOperators(self):  # {{{
    281   operators = {}
--> 283   version = parse_version(getCdoVersion(self.CDO))
    284   if (version < parse_version('1.7.2')):
    285     proc = subprocess.Popen([self.CDO, '-h'], stderr=subprocess.PIPE, stdout=subprocess.PIPE)

File ~/anaconda3/lib/python3.10/site-packages/cdo.py:78, in getCdoVersion(path2cdo, verbose)
     77 def getCdoVersion(path2cdo, verbose=False):
---> 78   proc = subprocess.Popen([path2cdo, '-V'], stderr=subprocess.PIPE, stdout=subprocess.PIPE)
     79   ret = proc.communicate()
     81   cdo_help_stdout = ret[0].decode("utf-8")

File ~/anaconda3/lib/python3.10/subprocess.py:971, in Popen.__init__(self, args, bufsize, executable, stdin, stdout, stderr, preexec_fn, close_fds, shell, cwd, env, universal_newlines, startupinfo, creationflags, restore_signals, start_new_session, pass_fds, user, group, extra_groups, encoding, errors, text, umask, pipesize)
    967         if self.text_mode:
    968             self.stderr = io.TextIOWrapper(self.stderr,
    969                     encoding=encoding, errors=errors)
--> 971     self._execute_child(args, executable, preexec_fn, close_fds,
    972                         pass_fds, cwd, env,
    973                         startupinfo, creationflags, shell,
    974                         p2cread, p2cwrite,
    975                         c2pread, c2pwrite,
    976                         errread, errwrite,
    977                         restore_signals,
    978                         gid, gids, uid, umask,
    979                         start_new_session)
    980 except:
    981     # Cleanup if the child failed starting.
    982     for f in filter(None, (self.stdin, self.stdout, self.stderr)):

File ~/anaconda3/lib/python3.10/subprocess.py:1722, in Popen._execute_child(self, args, executable, preexec_fn, close_fds, pass_fds, cwd, env, startupinfo, creationflags, shell, p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite, restore_signals, gid, gids, uid, umask, start_new_session)
   1717     executable = args[0]
   1719 sys.audit("subprocess.Popen", executable, args, cwd, env)
   1721 if (_USE_POSIX_SPAWN
-> 1722         and os.path.dirname(executable)
   1723         and preexec_fn is None
   1724         and not close_fds
   1725         and not pass_fds
   1726         and cwd is None
   1727         and (p2cread == -1 or p2cread > 2)
   1728         and (c2pwrite == -1 or c2pwrite > 2)
   1729         and (errwrite == -1 or errwrite > 2)
   1730         and not start_new_session
   1731         and gid is None
   1732         and gids is None
   1733         and uid is None
   1734         and umask < 0):
   1735     self._posix_spawn(args, executable, env, restore_signals,
   1736                       p2cread, p2cwrite,
   1737                       c2pread, c2pwrite,
   1738                       errread, errwrite)
   1739     return

File ~/anaconda3/lib/python3.10/posixpath.py:152, in dirname(p)
    150 def dirname(p):
    151     """Returns the directory component of a pathname""" 
--> 152     p = os.fspath(p)
    153     sep = _get_sep(p)
    154     i = p.rfind(sep) + 1

TypeError: expected str, bytes or os.PathLike object, not NoneType

My os is Ubuntu 22.04.2 LTS with x86-64 architecture.


Replies (2)

RE: Jupyter with CDO - Added by Ralf Mueller about 1 year ago

hi!
please upload the script you are using

normal usage is to first create a CDO object like

cdo = Cdo()
and perform subsequent methods calls on this object instead of Cdo()

Do you have installed the CDO binary?

RE: Jupyter with CDO - Added by Ilias Moysidis about 1 year ago

Sorry for the confusion. The problem was that the python executable and the jupyter kernel where in different environments. The kernel was running from the virtual environment of my project, while the executable was running from the conda environment. I fixed the issue by changing the interpreter of my local project to conda. To anyone who is interested, an explanation of the solution can be found at https://jakevdp.github.io/blog/2017/12/05/installing-python-packages-from-jupyter/.

    (1-2/2)