Commit 35f3e7c3 authored by Celine Mercier's avatar Celine Mercier

All commands now handle outputing to another DMS + small fixes

parent 8a8e9e50
......@@ -8,7 +8,7 @@ from obitools3.apps.optiongroups import addMinimalInputOption, addTaxonomyOption
from obitools3.dms.view import RollbackException
from functools import reduce
from obitools3.apps.config import logger
from obitools3.utils cimport tobytes
from obitools3.utils cimport tobytes, str2bytes
from obitools3.dms.capi.obiview cimport NUC_SEQUENCE_COLUMN, \
ID_COLUMN, \
DEFINITION_COLUMN, \
......@@ -272,24 +272,39 @@ def run(config):
input = open_uri(config['obi']['inputURI'])
if input is None:
raise Exception("Could not read input view")
i_dms = input[0]
i_view = input[1]
i_view_name = input[1].name
# Read the name of the output view
uri = config['obi']['outputURI'].split('/')
if len(uri)==2:
# Check that input and output DMS are the same (predicate, to discuss)
if config['obi']['inputURI'].split('/')[0] != uri[0]:
raise Exception("Input and output DMS must be the same")
output_view_name = uri[1]
else:
output_view_name = uri[0]
# Open the output: only the DMS, as the output view is going to be created by cloning the input view
# (could eventually be done via an open_uri() argument)
output = open_uri(config['obi']['outputURI'],
input=False,
dms_only=True)
if output is None:
raise Exception("Could not create output view")
o_dms = output[0]
o_view_name = output[1]
# If the input and output DMS are not the same, import the input view in the output DMS before cloning it to modify it
# (could be the other way around: clone and modify in the input DMS then import the new view in the output DMS)
if i_dms != o_dms:
imported_view_name = i_view_name
i=0
while imported_view_name in o_dms: # Making sure view name is unique in output DMS
imported_view_name = i_view_name+b"_"+str2bytes(str(i))
i+=1
View.import_view(i_dms.full_path[:-7], o_dms.full_path[:-7], i_view_name, imported_view_name)
i_view = o_dms[imported_view_name]
# Clone output view from input view
o_view = i_view.clone(output_view_name, comments=i_view.comments) # TODO comments
o_view = i_view.clone(o_view_name)
if o_view is None:
raise Exception("Couldn't create output view")
i_view.close()
if 'taxoURI' in config['obi'] : # TODO default None problem
# Open taxonomy if there is one
if 'taxoURI' in config['obi'] and config['obi']['taxoURI'] is not None:
taxo_uri = open_uri(config['obi']['taxoURI'])
if taxo_uri is None:
raise Exception("Couldn't open taxonomy")
......@@ -345,16 +360,19 @@ def run(config):
# Save command config in View and DMS comments
command_line = " ".join(sys.argv[1:])
input_dms_name=[input[0].name]
input_view_name=[input[1].name]
input_view_name=[i_view_name]
if 'taxoURI' in config['obi'] and config['obi']['taxoURI'] is not None:
input_dms_name.append(config['obi']['taxoURI'].split("/", 1)[0])
input_view_name.append(config['obi']['taxoURI'].split("/", 1)[1])
input_dms_name.append(config['obi']['taxoURI'].split("/")[-3])
input_view_name.append("taxonomy/"+config['obi']['taxoURI'].split("/")[-1])
o_view.write_config(config, "annotate", command_line, input_dms_name=input_dms_name, input_view_name=input_view_name)
input[0].record_command_line(command_line) # TODO assuming same dms
output[0].record_command_line(command_line)
print("\n")
print(repr(o_view))
input[0].close()
# output[0].close()
# If the input and the output DMS are different, delete the temporary imported view used to create the final view
if i_dms != o_dms:
View.delete_view(o_dms, imported_view_name)
o_dms.close()
i_dms.close()
......@@ -8,7 +8,7 @@ from obitools3.dms.capi.obiclean cimport obi_clean
from obitools3.apps.optiongroups import addMinimalInputOption, addMinimalOutputOption
from obitools3.uri.decode import open_uri
from obitools3.apps.config import logger
from obitools3.utils cimport tobytes
from obitools3.utils cimport tobytes, str2bytes
from obitools3.dms.view.view cimport View
from obitools3.dms.view.typed_view.view_NUC_SEQS cimport View_NUC_SEQS
......@@ -66,38 +66,58 @@ def run(config):
DMS.obi_atexit()
logger("info", "obi clean")
# Open the input: only the DMS
input = open_uri(config['obi']['inputURI'],
dms_only=True)
if input is None:
raise Exception("Could not read input")
i_dms = input[0]
i_dms_name = input[0].name
i_view_name = input[1]
# Open the output: only the DMS
output = open_uri(config['obi']['outputURI'],
input=False,
dms_only=True)
if output is None:
raise Exception("Could not create output")
o_dms = output[0]
final_o_view_name = output[1]
# Open DMS
dms_name = config['obi']['inputURI'].split('/')[0]
dms = open_uri(dms_name)[0]
# Read the name of the input view
uri_i = config['obi']['inputURI'].split('/')
i_view_name = uri_i[1]
# Read the name of the output view
uri_o = config['obi']['outputURI'].split('/')
if len(uri_o)==2:
# Check that input and output DMS are the same (predicate, to discuss)
if dms_name != uri_o[0]:
raise Exception("Input and output DMS must be the same")
o_view_name = uri_o[1]
# If the input and output DMS are not the same, run obiclean creating a temporary view that will be exported to
# the right DMS and deleted in the other afterwards.
if i_dms != o_dms:
temporary_view_name = final_o_view_name
i=0
while temporary_view_name in i_dms: # Making sure view name is unique in input DMS
temporary_view_name = final_o_view_name+b"_"+str2bytes(str(i))
i+=1
o_view_name = temporary_view_name
else:
o_view_name = uri_o[0]
o_view_name = final_o_view_name
# Save command config in View comments
command_line = " ".join(sys.argv[1:])
comments = View.print_config(config, "clean", command_line, input_dms_name=[dms_name], input_view_name=[i_view_name])
comments = View.print_config(config, "clean", command_line, input_dms_name=[i_dms_name], input_view_name=[i_view_name])
if obi_clean(tobytes(dms_name), tobytes(i_view_name), tobytes(config['clean']['sample-tag-name']), tobytes(o_view_name), comments, \
if obi_clean(tobytes(i_dms_name), tobytes(i_view_name), tobytes(config['clean']['sample-tag-name']), tobytes(o_view_name), comments, \
config['clean']['distance'], config['clean']['ratio'], config['clean']['heads-only'], 1) < 0:
raise Exception("Error running obiclean")
# If the input and output DMS are not the same, export result view to output DMS
if i_dms != o_dms:
View.import_view(i_dms.full_path[:-7], o_dms.full_path[:-7], o_view_name, final_o_view_name)
# Save command config in DMS comments
dms.record_command_line(command_line)
o_dms.record_command_line(command_line)
print("\n")
print(repr(dms[o_view_name]))
dms.close()
print(repr(o_dms[final_o_view_name]))
# If the input and the output DMS are different, delete the temporary result view in the input DMS
if i_dms != o_dms:
View.delete_view(i_dms, o_view_name)
o_dms.close()
i_dms.close()
......@@ -144,34 +144,37 @@ def run(config):
DMS.obi_atexit()
logger("info", "obi ecopcr")
# TODO Bad URI reading because current one is not adapted
# Get input DMS path
i_dms_name = config['obi']['inputURI'].split('/')[0]
# Read the name of the input view
i_uri = config['obi']['inputURI'].split('/')
i_view_name = i_uri[1]
# Read the name of the output view
o_uri = config['obi']['outputURI'].split('/')
if len(o_uri)==2:
# Get output DMS path
o_dms_name = o_uri[0]
o_view_name = o_uri[1]
else:
o_dms_name = i_dms_name
o_view_name = o_uri[0]
o_dms = open_uri(o_dms_name, input=False)[0]
# Read taxonomy name
taxonomy_name = config['obi']['taxoURI'].split('/')[2]
# Open the input: only the DMS
input = open_uri(config['obi']['inputURI'],
dms_only=True)
if input is None:
raise Exception("Could not read input")
i_dms = input[0]
i_dms_name = input[0].name
i_view_name = input[1]
# Open the output: only the DMS
output = open_uri(config['obi']['outputURI'],
input=False,
dms_only=True)
if output is None:
raise Exception("Could not create output")
o_dms = output[0]
o_dms_name = output[0].name
o_view_name = output[1]
# Read taxonomy name
taxonomy_name = config['obi']['taxoURI'].split("/")[-1] # Robust in theory
# Save command config in View comments
command_line = " ".join(sys.argv[1:])
comments = View.print_config(config, "ecopcr", command_line, input_dms_name=[i_dms_name], input_view_name=[i_view_name, config['obi']['taxoURI']])
input_dms_name=[i_dms_name]
input_view_name= [i_view_name]
input_dms_name.append(config['obi']['taxoURI'].split("/")[-3])
input_view_name.append("taxonomy/"+config['obi']['taxoURI'].split("/")[-1])
comments = View.print_config(config, "ecopcr", command_line, input_dms_name=input_dms_name, input_view_name=input_view_name)
# TODO: primers in comments?
......
......@@ -7,7 +7,7 @@ from obitools3.uri.decode import open_uri
from obitools3.apps.optiongroups import addMinimalInputOption, addTaxonomyOption, addMinimalOutputOption
from obitools3.dms.view import RollbackException
from obitools3.apps.config import logger
from obitools3.utils cimport tobytes
from obitools3.utils cimport tobytes, str2bytes
from functools import reduce
import time
......@@ -267,19 +267,28 @@ def run(config):
input = open_uri(config["obi"]["inputURI"])
if input is None:
raise Exception("Could not read input view")
i_dms = input[0]
i_view = input[1]
# Read the name of the output view
uri = config["obi"]["outputURI"].split("/")
if len(uri)==2:
# Check that input and output DMS are the same (predicate, to discuss)
if config["obi"]["inputURI"].split("/")[0] != uri[0]:
raise Exception("Input and output DMS must be the same")
output_view_name = uri[1]
else:
output_view_name = uri[0]
# Open the output: only the DMS
output = open_uri(config['obi']['outputURI'],
input=False,
dms_only=True)
if output is None:
raise Exception("Could not create output view")
o_dms = output[0]
o_view_name_final = output[1]
o_view_name = o_view_name_final
# If the input and output DMS are not the same, create output view in input DMS first, then export it
# to output DMS, making sure the temporary view name is unique in the input DMS
if i_dms != o_dms:
i=0
while o_view_name in i_dms:
o_view_name = o_view_name_final+b"_"+str2bytes(str(i))
i+=1
if "taxoURI" in config["obi"] : # TODO default None problem
if 'taxoURI' in config['obi'] and config['obi']['taxoURI'] is not None:
taxo_uri = open_uri(config["obi"]["taxoURI"])
if taxo_uri is None:
raise Exception("Couldn't open taxonomy")
......@@ -307,25 +316,32 @@ def run(config):
# Create output view with the line selection
try:
o_view = selection.materialize(output_view_name)
o_view = selection.materialize(o_view_name)
except Exception, e:
raise RollbackException("obi grep error, rollbacking view: "+str(e), o_view)
# TODO DISCUSS if output URI to different DMS, copy view?
# Save command config in View and DMS comments
command_line = " ".join(sys.argv[1:])
input_dms_name=[input[0].name]
input_view_name=[input[1].name]
if 'taxoURI' in config['obi'] and config['obi']['taxoURI'] is not None:
input_dms_name.append(config['obi']['taxoURI'].split("/", 1)[0])
input_view_name.append(config['obi']['taxoURI'].split("/", 1)[1])
input_dms_name.append(config['obi']['taxoURI'].split("/")[-3])
input_view_name.append("taxonomy/"+config['obi']['taxoURI'].split("/")[-1])
o_view.write_config(config, "grep", command_line, input_dms_name=input_dms_name, input_view_name=input_view_name)
input[0].record_command_line(command_line) # TODO assuming input and output dms are the same
o_dms.record_command_line(command_line)
# If input and output DMS are not the same, export the temporary view to the output DMS
# and delete the temporary view in the input DMS
if i_dms != o_dms:
o_view.close()
View.import_view(i_dms.full_path[:-7], o_dms.full_path[:-7], o_view_name, o_view_name_final)
o_view = o_dms[o_view_name_final]
print("\n")
print(repr(o_view))
input[0].close()
#output[0].close()
# If the input and the output DMS are different, delete the temporary imported view used to create the final view
if i_dms != o_dms:
View.delete_view(i_dms, o_view_name)
o_dms.close()
i_dms.close()
......@@ -7,6 +7,7 @@ from obitools3.uri.decode import open_uri
from obitools3.apps.optiongroups import addMinimalInputOption, addMinimalOutputOption
from obitools3.dms.view import RollbackException
from obitools3.apps.config import logger
from obitools3.utils cimport str2bytes
import time
import sys
......@@ -37,50 +38,65 @@ def run(config):
logger("info", "obi head")
# Open the input
input = open_uri(config['obi']['inputURI'])
input = open_uri(config["obi"]["inputURI"])
if input is None:
raise Exception("Could not read input view")
i_dms = input[0]
i_view = input[1]
# Read the name of the output view
uri = config['obi']['outputURI'].split('/')
if len(uri)==2:
# Check that input and output DMS are the same (predicate, to discuss)
if config['obi']['inputURI'].split('/')[0] != uri[0]:
raise Exception("Input and output DMS must be the same")
output_view_name = uri[1]
else:
output_view_name = uri[0]
# Initialize the progress bar
pb = ProgressBar(len(i_view), config, seconde=5)
# Open the output: only the DMS
output = open_uri(config['obi']['outputURI'],
input=False,
dms_only=True)
if output is None:
raise Exception("Could not create output view")
o_dms = output[0]
o_view_name_final = output[1]
o_view_name = o_view_name_final
# If the input and output DMS are not the same, create output view in input DMS first, then export it
# to output DMS, making sure the temporary view name is unique in the input DMS
if i_dms != o_dms:
i=0
while o_view_name in i_dms:
o_view_name = o_view_name_final+b"_"+str2bytes(str(i))
i+=1
n = min(config['head']['count'], len(i_view))
# Initialize the progress bar
pb = ProgressBar(n, config, seconde=5)
selection = Line_selection(i_view)
for i in range(n):
pb(i)
selection.append(i)
# Save command config in View comments
command_line = " ".join(sys.argv[1:])
comments = View.get_config_dict(config, "head", command_line, input_dms_name=[i_dms.name], input_view_name=[i_view.name])
# Create output view with the line selection
try:
o_view = selection.materialize(output_view_name, comments=comments)
o_view = selection.materialize(o_view_name)
except Exception, e:
raise RollbackException("obi head error, rollbacking view: "+str(e), o_view)
# TODO DISCUSS if output URI to different DMS, copy view?
# Save command config in DMS comments
i_dms.record_command_line(command_line)
command_line = " ".join(sys.argv[1:])
o_view.write_config(config, "head", command_line, input_dms_name=[i_dms.name], input_view_name=[i_view.name])
o_dms.record_command_line(command_line)
# If input and output DMS are not the same, export the temporary view to the output DMS
# and delete the temporary view in the input DMS
if i_dms != o_dms:
o_view.close()
View.import_view(i_dms.full_path[:-7], o_dms.full_path[:-7], o_view_name, o_view_name_final)
o_view = o_dms[o_view_name_final]
print("\n")
print(repr(o_view))
input[0].close()
#output[0].close()
# If the input and the output DMS are different, delete the temporary imported view used to create the final view
if i_dms != o_dms:
View.delete_view(i_dms, o_view_name)
o_dms.close()
i_dms.close()
......@@ -511,7 +511,7 @@ def run(config):
o_view.write_config(config, "ngsfilter", command_line, input_dms_name=input_dms_name, input_view_name=input_view_name)
unidentified.write_config(config, "ngsfilter", command_line, input_dms_name=input_dms_name, input_view_name=input_view_name)
# TODO add comment about unidentified seqs
output[0].record_command_line(command_line) # TODO if same dms...
output[0].record_command_line(command_line)
print("\n")
print(repr(o_view))
......
......@@ -7,6 +7,8 @@ from obitools3.uri.decode import open_uri
from obitools3.apps.optiongroups import addMinimalInputOption, addMinimalOutputOption
from obitools3.dms.view import RollbackException
from obitools3.apps.config import logger
from obitools3.utils cimport str2bytes
from obitools3.dms.capi.obitypes cimport OBI_BOOL, \
OBI_CHAR, \
OBI_FLOAT, \
......@@ -70,20 +72,29 @@ def run(config):
logger("info", "obi sort")
# Open the input
input = open_uri(config['obi']['inputURI'])
input = open_uri(config["obi"]["inputURI"])
if input is None:
raise Exception("Could not read input view")
i_dms = input[0]
i_view = input[1]
# Read the name of the output view
uri = config['obi']['outputURI'].split('/')
if len(uri)==2:
# Check that input and output DMS are the same (predicate, to discuss)
if config['obi']['inputURI'].split('/')[0] != uri[0]:
raise Exception("Input and output DMS must be the same")
output_view_name = uri[1]
else:
output_view_name = uri[0]
# Open the output: only the DMS
output = open_uri(config['obi']['outputURI'],
input=False,
dms_only=True)
if output is None:
raise Exception("Could not create output view")
o_dms = output[0]
o_view_name_final = output[1]
o_view_name = o_view_name_final
# If the input and output DMS are not the same, create output view in input DMS first, then export it
# to output DMS, making sure the temporary view name is unique in the input DMS
if i_dms != o_dms:
i=0
while o_view_name in i_dms:
o_view_name = o_view_name_final+b"_"+str2bytes(str(i))
i+=1
# Initialize the progress bar
pb = ProgressBar(len(i_view), config, seconde=5)
......@@ -100,20 +111,29 @@ def run(config):
# Create output view with the sorted line selection
try:
o_view = selection.materialize(output_view_name, comments="obi sort: "+str(config['sort']['keys'])+"\n")
o_view = selection.materialize(o_view_name)
except Exception, e:
raise RollbackException("obi sort error, rollbacking view: "+str(e), o_view)
# TODO DISCUSS if output URI to different DMS, copy view?
# Save command config in View and DMS comments
command_line = " ".join(sys.argv[1:])
o_view.write_config(config, "sort", command_line, input_dms_name=[input[0].name], input_view_name=[input[1].name])
input[0].record_command_line(command_line) # TODO assuming same dms...
input_dms_name=[input[0].name]
input_view_name=[input[1].name]
o_view.write_config(config, "sort", command_line, input_dms_name=input_dms_name, input_view_name=input_view_name)
o_dms.record_command_line(command_line)
# If input and output DMS are not the same, export the temporary view to the output DMS
# and delete the temporary view in the input DMS
if i_dms != o_dms:
o_view.close()
View.import_view(i_dms.full_path[:-7], o_dms.full_path[:-7], o_view_name, o_view_name_final)
o_view = o_dms[o_view_name_final]
print("\n")
print(repr(o_view))
input[0].close()
#output[0].close()
# If the input and the output DMS are different, delete the temporary imported view used to create the final view
if i_dms != o_dms:
View.delete_view(i_dms, o_view_name)
o_dms.close()
i_dms.close()
......@@ -143,7 +143,7 @@ def run(config):
raise Exception("Could not read input view")
i_view = input[1]
if 'taxoURI' in config['obi'] : # TODO default None problem
if 'taxoURI' in config['obi'] and config['obi']['taxoURI'] is not None:
taxo_uri = open_uri(config['obi']['taxoURI'])
if taxo_uri is None:
raise Exception("Couldn't open taxonomy")
......
......@@ -7,6 +7,7 @@ from obitools3.uri.decode import open_uri
from obitools3.apps.optiongroups import addMinimalInputOption, addMinimalOutputOption
from obitools3.dms.view import RollbackException
from obitools3.apps.config import logger
from obitools3.utils cimport str2bytes
import time
import sys
......@@ -37,30 +38,39 @@ def run(config):
logger("info", "obi tail")
# Open the input
input = open_uri(config['obi']['inputURI'])
input = open_uri(config["obi"]["inputURI"])
if input is None:
raise Exception("Could not read input view")
i_dms = input[0]
i_view = input[1]
# Read the name of the output view
uri = config['obi']['outputURI'].split('/')
if len(uri)==2:
# Check that input and output DMS are the same (predicate, to discuss)
if config['obi']['inputURI'].split('/')[0] != uri[0]:
raise Exception("Input and output DMS must be the same")
output_view_name = uri[1]
else:
output_view_name = uri[0]
# Initialize the progress bar
pb = ProgressBar(len(i_view), config, seconde=5)
# Open the output: only the DMS
output = open_uri(config['obi']['outputURI'],
input=False,
dms_only=True)
if output is None:
raise Exception("Could not create output view")
o_dms = output[0]
o_view_name_final = output[1]
o_view_name = o_view_name_final
# If the input and output DMS are not the same, create output view in input DMS first, then export it
# to output DMS, making sure the temporary view name is unique in the input DMS
if i_dms != o_dms:
i=0
while o_view_name in i_dms:
o_view_name = o_view_name_final+b"_"+str2bytes(str(i))
i+=1
start = max(len(i_view) - config['tail']['count'], 0)
# Initialize the progress bar
pb = ProgressBar(len(i_view) - start, config, seconde=5)
selection = Line_selection(i_view)
for i in range(start, len(i_view)):
pb(i)