Merge branch 'dstftw-compat-getenv-and-expanduser'

This commit is contained in:
Sergey M․ 2014-10-26 19:58:39 +07:00
commit 639a422d21
6 changed files with 110 additions and 13 deletions

View File

@ -45,6 +45,9 @@ from youtube_dl.utils import (
escape_rfc3986, escape_rfc3986,
escape_url, escape_url,
js_to_json, js_to_json,
get_filesystem_encoding,
compat_getenv,
compat_expanduser,
) )
@ -355,5 +358,15 @@ class TestUtil(unittest.TestCase):
on = js_to_json('{"abc": true}') on = js_to_json('{"abc": true}')
self.assertEqual(json.loads(on), {'abc': True}) self.assertEqual(json.loads(on), {'abc': True})
def test_compat_getenv(self):
test_str = 'тест'
os.environ['YOUTUBE-DL-TEST'] = test_str.encode(get_filesystem_encoding())
self.assertEqual(compat_getenv('YOUTUBE-DL-TEST'), test_str)
def test_compat_expanduser(self):
test_str = 'C:\Documents and Settings\тест\Application Data'
os.environ['HOME'] = test_str.encode(get_filesystem_encoding())
self.assertEqual(compat_expanduser('~'), test_str)
if __name__ == '__main__': if __name__ == '__main__':
unittest.main() unittest.main()

View File

@ -24,6 +24,7 @@ if os.name == 'nt':
from .utils import ( from .utils import (
compat_cookiejar, compat_cookiejar,
compat_expanduser,
compat_http_client, compat_http_client,
compat_str, compat_str,
compat_urllib_error, compat_urllib_error,
@ -451,7 +452,7 @@ class YoutubeDL(object):
template_dict = collections.defaultdict(lambda: 'NA', template_dict) template_dict = collections.defaultdict(lambda: 'NA', template_dict)
outtmpl = self.params.get('outtmpl', DEFAULT_OUTTMPL) outtmpl = self.params.get('outtmpl', DEFAULT_OUTTMPL)
tmpl = os.path.expanduser(outtmpl) tmpl = compat_expanduser(outtmpl)
filename = tmpl % template_dict filename = tmpl % template_dict
return filename return filename
except ValueError as err: except ValueError as err:

View File

@ -98,6 +98,7 @@ from .options import (
parseOpts, parseOpts,
) )
from .utils import ( from .utils import (
compat_expanduser,
compat_getpass, compat_getpass,
compat_print, compat_print,
DateRange, DateRange,
@ -287,7 +288,7 @@ def _real_main(argv=None):
u' template'.format(outtmpl)) u' template'.format(outtmpl))
any_printing = opts.geturl or opts.gettitle or opts.getid or opts.getthumbnail or opts.getdescription or opts.getfilename or opts.getformat or opts.getduration or opts.dumpjson or opts.dump_single_json any_printing = opts.geturl or opts.gettitle or opts.getid or opts.getthumbnail or opts.getdescription or opts.getfilename or opts.getformat or opts.getduration or opts.dumpjson or opts.dump_single_json
download_archive_fn = os.path.expanduser(opts.download_archive) if opts.download_archive is not None else opts.download_archive download_archive_fn = compat_expanduser(opts.download_archive) if opts.download_archive is not None else opts.download_archive
ydl_opts = { ydl_opts = {
'usenetrc': opts.usenetrc, 'usenetrc': opts.usenetrc,

View File

@ -9,6 +9,7 @@ import shutil
import traceback import traceback
from .utils import ( from .utils import (
compat_expanduser,
write_json_file, write_json_file,
) )
@ -22,7 +23,7 @@ class Cache(object):
if res is None: if res is None:
cache_root = os.environ.get('XDG_CACHE_HOME', '~/.cache') cache_root = os.environ.get('XDG_CACHE_HOME', '~/.cache')
res = os.path.join(cache_root, 'youtube-dl') res = os.path.join(cache_root, 'youtube-dl')
return os.path.expanduser(res) return compat_expanduser(res)
def _get_cache_fn(self, section, key, dtype): def _get_cache_fn(self, section, key, dtype):
assert re.match(r'^[a-zA-Z0-9_.-]+$', section), \ assert re.match(r'^[a-zA-Z0-9_.-]+$', section), \

View File

@ -6,6 +6,8 @@ import shlex
import sys import sys
from .utils import ( from .utils import (
compat_expanduser,
compat_getenv,
get_term_width, get_term_width,
write_string, write_string,
) )
@ -27,19 +29,19 @@ def parseOpts(overrideArguments=None):
return res return res
def _readUserConf(): def _readUserConf():
xdg_config_home = os.environ.get('XDG_CONFIG_HOME') xdg_config_home = compat_getenv('XDG_CONFIG_HOME')
if xdg_config_home: if xdg_config_home:
userConfFile = os.path.join(xdg_config_home, 'youtube-dl', 'config') userConfFile = os.path.join(xdg_config_home, 'youtube-dl', 'config')
if not os.path.isfile(userConfFile): if not os.path.isfile(userConfFile):
userConfFile = os.path.join(xdg_config_home, 'youtube-dl.conf') userConfFile = os.path.join(xdg_config_home, 'youtube-dl.conf')
else: else:
userConfFile = os.path.join(os.path.expanduser('~'), '.config', 'youtube-dl', 'config') userConfFile = os.path.join(compat_expanduser('~'), '.config', 'youtube-dl', 'config')
if not os.path.isfile(userConfFile): if not os.path.isfile(userConfFile):
userConfFile = os.path.join(os.path.expanduser('~'), '.config', 'youtube-dl.conf') userConfFile = os.path.join(compat_expanduser('~'), '.config', 'youtube-dl.conf')
userConf = _readOptions(userConfFile, None) userConf = _readOptions(userConfFile, None)
if userConf is None: if userConf is None:
appdata_dir = os.environ.get('appdata') appdata_dir = compat_getenv('appdata')
if appdata_dir: if appdata_dir:
userConf = _readOptions( userConf = _readOptions(
os.path.join(appdata_dir, 'youtube-dl', 'config'), os.path.join(appdata_dir, 'youtube-dl', 'config'),
@ -51,11 +53,11 @@ def parseOpts(overrideArguments=None):
if userConf is None: if userConf is None:
userConf = _readOptions( userConf = _readOptions(
os.path.join(os.path.expanduser('~'), 'youtube-dl.conf'), os.path.join(compat_expanduser('~'), 'youtube-dl.conf'),
default=None) default=None)
if userConf is None: if userConf is None:
userConf = _readOptions( userConf = _readOptions(
os.path.join(os.path.expanduser('~'), 'youtube-dl.conf.txt'), os.path.join(compat_expanduser('~'), 'youtube-dl.conf.txt'),
default=None) default=None)
if userConf is None: if userConf is None:

View File

@ -203,6 +203,82 @@ def compat_ord(c):
if type(c) is int: return c if type(c) is int: return c
else: return ord(c) else: return ord(c)
if sys.version_info >= (3, 0):
compat_getenv = os.getenv
compat_expanduser = os.path.expanduser
else:
# Environment variables should be decoded with filesystem encoding.
# Otherwise it will fail if any non-ASCII characters present (see #3854 #3217 #2918)
def compat_getenv(key, default=None):
env = os.getenv(key, default)
if env:
env = env.decode(get_filesystem_encoding())
return env
# HACK: The default implementations of os.path.expanduser from cpython do not decode
# environment variables with filesystem encoding. We will work around this by
# providing adjusted implementations.
# The following are os.path.expanduser implementations from cpython 2.7.8 stdlib
# for different platforms with correct environment variables decoding.
if os.name == 'posix':
def compat_expanduser(path):
"""Expand ~ and ~user constructions. If user or $HOME is unknown,
do nothing."""
if not path.startswith('~'):
return path
i = path.find('/', 1)
if i < 0:
i = len(path)
if i == 1:
if 'HOME' not in os.environ:
import pwd
userhome = pwd.getpwuid(os.getuid()).pw_dir
else:
userhome = compat_getenv('HOME')
else:
import pwd
try:
pwent = pwd.getpwnam(path[1:i])
except KeyError:
return path
userhome = pwent.pw_dir
userhome = userhome.rstrip('/')
return (userhome + path[i:]) or '/'
elif os.name == 'nt' or os.name == 'ce':
def compat_expanduser(path):
"""Expand ~ and ~user constructs.
If user or $HOME is unknown, do nothing."""
if path[:1] != '~':
return path
i, n = 1, len(path)
while i < n and path[i] not in '/\\':
i = i + 1
if 'HOME' in os.environ:
userhome = compat_getenv('HOME')
elif 'USERPROFILE' in os.environ:
userhome = compat_getenv('USERPROFILE')
elif not 'HOMEPATH' in os.environ:
return path
else:
try:
drive = compat_getenv('HOMEDRIVE')
except KeyError:
drive = ''
userhome = os.path.join(drive, compat_getenv('HOMEPATH'))
if i != 1: #~user
userhome = os.path.join(os.path.dirname(userhome), path[1:i])
return userhome + path[i:]
else:
compat_expanduser = os.path.expanduser
# This is not clearly defined otherwise # This is not clearly defined otherwise
compiled_regex_type = type(re.compile('')) compiled_regex_type = type(re.compile(''))
@ -1207,11 +1283,14 @@ class locked_file(object):
return self.f.read(*args) return self.f.read(*args)
def get_filesystem_encoding():
encoding = sys.getfilesystemencoding()
return encoding if encoding is not None else 'utf-8'
def shell_quote(args): def shell_quote(args):
quoted_args = [] quoted_args = []
encoding = sys.getfilesystemencoding() encoding = get_filesystem_encoding()
if encoding is None:
encoding = 'utf-8'
for a in args: for a in args:
if isinstance(a, bytes): if isinstance(a, bytes):
# We may get a filename encoded with 'encodeFilename' # We may get a filename encoded with 'encodeFilename'
@ -1261,7 +1340,7 @@ def format_bytes(bytes):
def get_term_width(): def get_term_width():
columns = os.environ.get('COLUMNS', None) columns = compat_getenv('COLUMNS', None)
if columns: if columns:
return int(columns) return int(columns)