Skip to content
Projects
Groups
Snippets
Help
This project
Loading...
Sign in / Register
Toggle navigation
O
OBITools3
Overview
Overview
Details
Activity
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
21
Issues
21
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
Wiki
Wiki
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Commits
Issue Boards
Open sidebar
OBITools
OBITools3
Commits
3cfe3a9b
Commit
3cfe3a9b
authored
Aug 29, 2019
by
Celine Mercier
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Improved progress display when importing files in a DMS
parent
728af51c
Hide whitespace changes
Inline
Side-by-side
Showing
5 changed files
with
36 additions
and
10 deletions
+36
-10
import.pyx
python/obitools3/commands/import.pyx
+8
-3
uncompress.pxd
python/obitools3/files/uncompress.pxd
+2
-0
uncompress.pyx
python/obitools3/files/uncompress.pyx
+15
-4
genbank.pyx
python/obitools3/parsers/genbank.pyx
+6
-2
utils.pyx
python/obitools3/utils.pyx
+5
-1
No files found.
python/obitools3/commands/import.pyx
View file @
3cfe3a9b
...
...
@@ -99,6 +99,7 @@ def run(config):
logger("info", "obi import: imports an object (file(s), obiview, taxonomy...) into a DMS")
entry_count = -1
pb = None
if not config['obi']['taxdump']:
input = open_uri(config['obi']['inputURI'])
...
...
@@ -110,7 +111,10 @@ def run(config):
else:
entry_count = input[4]
logger("info", "Importing %d entries", entry_count)
if entry_count > 0:
logger("info", "Importing %d entries", entry_count)
else:
logger("info", "Importing an unknow number of entries")
# TODO a bit dirty?
if input[2]==Nuc_Seq or input[2]==View_NUC_SEQS:
...
...
@@ -137,8 +141,6 @@ def run(config):
if entry_count >= 0:
pb = ProgressBar(entry_count, config, seconde=5)
else:
pb = None
entries = input[1]
...
...
@@ -170,6 +172,9 @@ def run(config):
if pb is not None:
pb(i)
elif not i%50000:
logger("info", "Imported %d entries", i)
if NUC_SEQS_view:
id_col[i] = entry.id
...
...
python/obitools3/files/uncompress.pxd
View file @
3cfe3a9b
...
...
@@ -15,4 +15,5 @@ cdef class MagicKeyFile:
cdef class CompressedFile:
cdef object accessor
cdef bint compressed
\ No newline at end of file
python/obitools3/files/uncompress.pyx
View file @
3cfe3a9b
...
...
@@ -74,8 +74,7 @@ cdef class MagicKeyFile:
cdef
class
CompressedFile
:
def
__init__
(
self
,
stream
)
:
cdef
int
keylength
cdef
MagicKeyFile
magic
...
...
@@ -92,11 +91,13 @@ cdef class CompressedFile:
magic
=
MagicKeyFile
(
stream
,
keylength
)
self
.
accessor
=
None
self
.
compressed
=
False
for
compressor
in
compress
:
k
,
c
=
compress
[
compressor
]
if
magic
.
key
.
startswith
(
k
)
:
self
.
accessor
=
c
(
magic
)
self
.
compressed
=
True
if
self
.
accessor
is
None
:
if
'b'
in
magic
.
stream_mode
:
...
...
@@ -110,7 +111,17 @@ cdef class CompressedFile:
'b'
not
in
magic
.
stream_mode
)
:
self
.
accessor
=
io
.
TextIOWrapper
(
self
.
accessor
)
# compressed property getter
@property
def
compressed
(
self
)
:
'''
Returns
a
boolean
indicating
whether
the
file
is
compressed
@rtype
:
bint
'''
return
self
.
compressed
def
__getattr__
(
self
,
name
)
:
return
getattr
(
self
.
accessor
,
name
)
...
...
python/obitools3/parsers/genbank.pyx
View file @
3cfe3a9b
...
...
@@ -166,9 +166,12 @@ def genbankIterator_dir(dir_path,
):
path = dir_path
read = 0
for filename in glob.glob(os.path.join(path, b'*.gbff*')):
read_files = 0
files = [filename for filename in glob.glob(os.path.join(path, b'*.gbff*'))]
for filename in files:
if read==only:
return
print("Parsing file %s (%d/%d)" % (tostr(filename), read_files, len(files)))
f = uopen(filename)
if only is not None:
only_f = only-read
...
...
@@ -177,7 +180,8 @@ def genbankIterator_dir(dir_path,
for seq in genbankIterator_file(f, skip=skip, only=only_f, buffersize=buffersize):
yield seq
read+=1
read_files+=1
def genbankIterator(obj,
int skip=0,
...
...
python/obitools3/utils.pyx
View file @
3cfe3a9b
...
...
@@ -15,6 +15,8 @@ from obitools3.dms.capi.obierrno cimport OBI_LINE_IDX_ERROR, \
OBI_ELT_IDX_ERROR, \
obi_errno
from obitools3.files.uncompress cimport CompressedFile
import re
import mmap
import os
...
...
@@ -45,7 +47,7 @@ cpdef int count_entries(file, bytes format):
return -1
sep = re.compile(sep)
if type(file) and (format == b'genbank' or format == b'embl'): # file is actually a directory with multiple files
if type(file)
== bytes
and (format == b'genbank' or format == b'embl'): # file is actually a directory with multiple files
files = []
if format == b'embl':
extensions = [b"*.dat"]
...
...
@@ -66,6 +68,8 @@ cpdef int count_entries(file, bytes format):
total_count = 0
for f in files:
if type(f) == CompressedFile and f.compressed:
return -1
mmapped_file = mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ)
total_count += len(re.findall(sep, mmapped_file))
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment