Skip to content

import_util

Author: Heli Qi Affiliation: NAIST Date: 2022.07

get_idle_gpu(ngpu=1, id_only=False)

Find idle GPUs for distributed learning.

Source code in speechain/utilbox/import_util.py 
37
38
39
40
41
42
43
44
45
46
47
48
49
50
def get_idle_gpu(ngpu: int = 1, id_only: bool = False) -> List[GPU]:
    """Find idle GPUs for distributed learning."""
    sorted_gpus = sorted(getGPUs(), key=lambda g: g.memoryUtil)
    if len(sorted_gpus) < ngpu:
        warnings.warn(
            f"Your machine doesn't have enough GPUs ({len(sorted_gpus)}) as you specified ({ngpu})! "
            f"Currently, only {len(sorted_gpus)} GPUs are used."
        )
    sorted_gpus = sorted_gpus[:ngpu]

    if id_only:
        return [gpu.id for gpu in sorted_gpus]
    else:
        return sorted_gpus

get_idle_port()

Find an idle port to used for distributed learning.

Source code in speechain/utilbox/import_util.py 
25
26
27
28
29
30
31
32
33
34
def get_idle_port() -> str:
    """Find an idle port to used for distributed learning."""
    pscmd = "netstat -ntl |grep -v Active| grep -v Proto|awk '{print $4}'|awk -F: '{print $NF}'"
    procs = os.popen(pscmd).read()
    procarr = procs.split("\n")
    tt = str(random.randint(15000, 30000))
    if tt not in procarr:
        return tt
    else:
        return get_idle_port()

parse_path_args(input_path)

This function parses the input path string into valid path before its later usage.

Parameters:

Name Type Description Default
input_path str

str If input_path starts with '/', no parsing will be done; Otherwise, this function will return its absolute path in the toolkit root, i.e., SPEECHAIN_ROOT.

 required

str

Type Description
str

Parsed valid absolute path.
Source code in speechain/utilbox/import_util.py 
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
def parse_path_args(input_path: str) -> str:
    """This function parses the input path string into valid path before its later
    usage.

    Args:
        input_path: str
            If input_path starts with '/', no parsing will be done;
            Otherwise, this function will return its absolute path in the toolkit root, i.e., SPEECHAIN_ROOT.

    Returns: str
        Parsed valid absolute path.
    """

    # do nothing for the absolute path
    if input_path.startswith("/"):
        return input_path

    # turn the general relative path into its absolute value
    elif input_path.startswith("."):
        return os.path.abspath(input_path)

    # turn the in-toolkit relative path into its absolute value
    else:
        assert "SPEECHAIN_ROOT" in os.environ.keys(), (
            "SPEECHAIN_ROOT doesn't exist in your environmental variables! "
            "Please move to the toolkit root and execute envir_preparation.sh there to build the toolkit environment!"
        )
        return os.path.join(os.environ["SPEECHAIN_ROOT"], input_path)