ndio/remote/ndingest.py
def verify_path(self, data, verifytype):
        """
        Verify the path supplied.
        """
        # Insert try and catch blocks
        try:
            token_name = data["project"]["token_name"]
        except:
            token_name = data["project"]["project_name"]

        channel_names = list(data["channels"].copy().keys())
        imgsz = data['dataset']['imagesize']

        for i in range(0, len(channel_names)):
            channel_type = data["channels"][
                channel_names[i]]["channel_type"]
            path = data["channels"][channel_names[i]]["data_url"]
            aws_pattern = re.compile("^(http:\/\/)(.+)(\.s3\.amazonaws\.com)")
            file_type = data["channels"][channel_names[i]]["file_type"]
            if "offset" in data["dataset"]:
                offset = data["dataset"]["offset"][0]
            else:
                offset = 0

            if (aws_pattern.match(path)):
                verifytype = VERIFY_BY_SLICE

            if (channel_type == "timeseries"):
                timerange = data["dataset"]["timerange"]
                try:
                    assert(timerange[0] != timerange[1])
                except AssertionError:
                    raise ValueError('Timeseries values are the same, did you\
specify the time steps?')
                for j in range(timerange[0], timerange[1] + 1):
                    # Test for tifs or such? Currently test for just not
                    # empty
                    if (verifytype == VERIFY_BY_FOLDER):
                        work_path = "{}/{}/{}/time{}/".format(
                            path, token_name, channel_names[i], ("%04d" % j))
                    elif (verifytype == VERIFY_BY_SLICE):
                        work_path = "{}/{}/{}/time{}/{}.{}".format(
                            path, token_name, channel_names[i], ("%04d" % j),
                            ("%04d" % offset), file_type)
                    else:
                        raise TypeError('Incorrect verify method')
                    # Check for accessibility
                    try:
                        if (verifytype == VERIFY_BY_FOLDER):
                            resp = requests.head(work_path)
                            assert(resp.status_code == 200)
                        elif (verifytype == VERIFY_BY_SLICE):
                            resp = requests.get(
                                work_path, stream=True, verify=False)
                            with open('/tmp/img.{}'.format(file_type),
                                      'wb') as out_file:
                                shutil.copyfileobj(resp.raw, out_file)
                            out_file.close()
                            assert(resp.status_code == 200)
                            resp.close()
                    except AssertionError:
                        raise OSError('Files are not http accessible: \
                            Error: {}, Path: {}'.format(resp.status_code,
                                                        work_path))
                    # Attempt to Verify imagesize here

                    try:
                        if (verifytype == VERIFY_BY_SLICE):
                            assert(list(self.identify_imagesize(file_type)) ==
                                   imgsz[0:2])
                    except:
                        raise ValueError('File image size does not match\
provided image size.')

            else:
                # Test for tifs or such? Currently test for just not empty
                if (verifytype == VERIFY_BY_FOLDER):
                    work_path = "{}/{}/{}/".format(
                        path, token_name, channel_names[i])
                elif (verifytype == VERIFY_BY_SLICE):
                    work_path = "{}/{}/{}/{}.{}".format(
                        path, token_name, channel_names[i],
                        ("%04d" % offset), file_type)
                else:
                    raise TypeError('Incorrect verify method')
                # Check for accessibility
                if (verifytype == VERIFY_BY_FOLDER):
                    resp = requests.head(work_path)
                elif (verifytype == VERIFY_BY_SLICE):
                    resp = requests.get(work_path, stream=True, verify=False)
                    with open('/tmp/img.{}'.format(file_type),
                              'wb') as out_file:
                        shutil.copyfileobj(resp.raw, out_file)
                    out_file.close()
                    resp.close()
                if (resp.status_code >= 300):
                    raise OSError('Files are not http accessible: \
                            Error: {}, Path: {}'.format(resp.status_code,
                                                        work_path))
                # Attempt to Verify imagesize here

                try:
                    if (verifytype == VERIFY_BY_SLICE):
                        assert(list(self.identify_imagesize(file_type)) ==
                               imgsz[0:2])
                except:
                    raise ValueError('File image size does not match\
provided image size.')
def verify_path(self, data, verifytype):
        """
        Verify the path supplied.
        """
        # Insert try and catch blocks
        try:
            token_name = data["project"]["token_name"]
        except:
            token_name = data["project"]["project_name"]

        channel_names = list(data["channels"].copy().keys())
        imgsz = data['dataset']['imagesize']

        for i in range(0, len(channel_names)):
            channel_type = data["channels"][
                channel_names[i]]["channel_type"]
            path = data["channels"][channel_names[i]]["data_url"]
            aws_pattern = re.compile("^(http:\/\/)(.+)(\.s3\.amazonaws\.com)")
            file_type = data["channels"][channel_names[i]]["file_type"]
            if "offset" in data["dataset"]:
                offset = data["dataset"]["offset"][0]
            else:
                offset = 0

            if (aws_pattern.match(path)):
                verifytype = VERIFY_BY_SLICE

            if (channel_type == "timeseries"):
                timerange = data["dataset"]["timerange"]
                try:
                    assert(timerange[0] != timerange[1])
                except AssertionError:
                    raise ValueError('Timeseries values are the same, did you\
specify the time steps?')
                for j in range(timerange[0], timerange[1] + 1):
                    # Test for tifs or such? Currently test for just not
                    # empty
                    if (verifytype == VERIFY_BY_FOLDER):
                        work_path = "{}/{}/{}/time{}/".format(
                            path, token_name, channel_names[i], ("%04d" % j))
                    elif (verifytype == VERIFY_BY_SLICE):
                        work_path = "{}/{}/{}/time{}/{}.{}".format(
                            path, token_name, channel_names[i], ("%04d" % j),
                            ("%04d" % offset), file_type)
                    else:
                        raise TypeError('Incorrect verify method')
                    # Check for accessibility
                    try:
                        if (verifytype == VERIFY_BY_FOLDER):
                            resp = requests.head(work_path)
                            assert(resp.status_code == 200)
                        elif (verifytype == VERIFY_BY_SLICE):
                            resp = requests.get(
                                work_path, stream=True, verify=False)
                            with open('/tmp/img.{}'.format(file_type),
                                      'wb') as out_file:
                                shutil.copyfileobj(resp.raw, out_file)
                            out_file.close()
                            assert(resp.status_code == 200)
                            resp.close()
                    except AssertionError:
                        raise OSError('Files are not http accessible: \
                            Error: {}, Path: {}'.format(resp.status_code,
                                                        work_path))
                    # Attempt to Verify imagesize here

                    try:
                        if (verifytype == VERIFY_BY_SLICE):
                            assert(list(self.identify_imagesize(file_type)) ==
                                   imgsz[0:2])
                    except:
                        raise ValueError('File image size does not match\
provided image size.')

            else:
                # Test for tifs or such? Currently test for just not empty
                if (verifytype == VERIFY_BY_FOLDER):
                    work_path = "{}/{}/{}/".format(
                        path, token_name, channel_names[i])
                elif (verifytype == VERIFY_BY_SLICE):
                    work_path = "{}/{}/{}/{}.{}".format(
                        path, token_name, channel_names[i],
                        ("%04d" % offset), file_type)
                else:
                    raise TypeError('Incorrect verify method')
                # Check for accessibility
                if (verifytype == VERIFY_BY_FOLDER):
                    resp = requests.head(work_path)
                elif (verifytype == VERIFY_BY_SLICE):
                    resp = requests.get(work_path, stream=True, verify=False)
                    with open('/tmp/img.{}'.format(file_type),
                              'wb') as out_file:
                        shutil.copyfileobj(resp.raw, out_file)
                    out_file.close()
                    resp.close()
                if (resp.status_code >= 300):
                    raise OSError('Files are not http accessible: \
                            Error: {}, Path: {}'.format(resp.status_code,
                                                        work_path))
                # Attempt to Verify imagesize here

                try:
                    if (verifytype == VERIFY_BY_SLICE):
                        assert(list(self.identify_imagesize(file_type)) ==
                               imgsz[0:2])
                except:
                    raise ValueError('File image size does not match\
provided image size.')
Verify the path supplied.