OR-Tools  8.2
file_util.cc
Go to the documentation of this file.
1// Copyright 2010-2018 Google LLC
2// Licensed under the Apache License, Version 2.0 (the "License");
3// you may not use this file except in compliance with the License.
4// You may obtain a copy of the License at
5//
6// http://www.apache.org/licenses/LICENSE-2.0
7//
8// Unless required by applicable law or agreed to in writing, software
9// distributed under the License is distributed on an "AS IS" BASIS,
10// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11// See the License for the specific language governing permissions and
12// limitations under the License.
13
15
16#include "absl/status/statusor.h"
17#include "absl/strings/str_cat.h"
18#include "google/protobuf/descriptor.h"
19#include "google/protobuf/io/zero_copy_stream_impl_lite.h"
20#include "google/protobuf/message.h"
21#include "google/protobuf/text_format.h"
22#include "google/protobuf/util/json_util.h"
23#include "ortools/base/file.h"
27
28namespace operations_research {
29
30using ::google::protobuf::TextFormat;
31
32absl::StatusOr<std::string> ReadFileToString(absl::string_view filename) {
33 std::string contents;
34 RETURN_IF_ERROR(file::GetContents(filename, &contents, file::Defaults()));
35 // Try decompressing it.
36 {
37 std::string uncompressed;
38 if (GunzipString(contents, &uncompressed)) contents.swap(uncompressed);
39 }
40 return contents;
41}
42
43bool ReadFileToProto(absl::string_view filename,
44 google::protobuf::Message* proto) {
45 std::string data;
46 CHECK_OK(file::GetContents(filename, &data, file::Defaults()));
47 // Try decompressing it.
48 {
49 std::string uncompressed;
50 if (GunzipString(data, &uncompressed)) {
51 VLOG(1) << "ReadFileToProto(): input is gzipped";
52 data.swap(uncompressed);
53 }
54 }
55 // Try binary format first, then text format, then JSON, then proto3 JSON,
56 // then give up.
57 // For some of those, like binary format and proto3 JSON, we perform
58 // additional checks to verify that we have the right proto: it can happen
59 // to try to read a proto of type Foo as a proto of type Bar, by mistake, and
60 // we'd rather have this function fail rather than silently accept it, because
61 // the proto parser is too lenient with unknown fields.
62 // We don't require ByteSizeLong(parsed) == input.size(), because it may be
63 // the case that the proto version changed and some fields are dropped.
64 // We just fail when the difference is too large.
65 constexpr double kMaxBinaryProtoParseShrinkFactor = 2;
66 if (proto->ParseFromString(data)) {
67 // NOTE(user): When using ParseFromString() from a generic
68 // google::protobuf::Message, like we do here, all fields are stored, even
69 // if their are unknown in the underlying proto type. Unless we explicitly
70 // discard those 'unknown fields' here, our call to ByteSizeLong() will
71 // still count the unknown payload.
72 proto->DiscardUnknownFields();
73 if (proto->ByteSizeLong() <
74 data.size() / kMaxBinaryProtoParseShrinkFactor) {
75 VLOG(1) << "ReadFileToProto(): input may be a binary proto, but of a "
76 "different proto";
77 } else {
78 VLOG(1) << "ReadFileToProto(): input seems to be a binary proto";
79 return true;
80 }
81 }
82 if (google::protobuf::TextFormat::ParseFromString(data, proto)) {
83 VLOG(1) << "ReadFileToProto(): input is a text proto";
84 return true;
85 }
86 if (google::protobuf::util::JsonStringToMessage(
87 data, proto, google::protobuf::util::JsonParseOptions())
88 .ok()) {
89 constexpr int kMaxJsonToBinaryShrinkFactor = 30;
90 if (proto->ByteSizeLong() < data.size() / kMaxJsonToBinaryShrinkFactor) {
91 VLOG(1) << "ReadFileToProto(): input is probably JSON, but probably not"
92 " of the right proto";
93 } else {
94 VLOG(1) << "ReadFileToProto(): input is a proto JSON";
95 return true;
96 }
97 }
98 LOG(WARNING) << "Could not parse protocol buffer";
99 return false;
100}
101
102bool WriteProtoToFile(absl::string_view filename,
103 const google::protobuf::Message& proto,
104 ProtoWriteFormat proto_write_format, bool gzipped,
105 bool append_extension_to_file_name) {
106 std::string file_type_suffix;
107 std::string output_string;
108 google::protobuf::io::StringOutputStream stream(&output_string);
109 switch (proto_write_format) {
111 if (!proto.SerializeToZeroCopyStream(&stream)) {
112 LOG(WARNING) << "Serialize to stream failed.";
113 return false;
114 }
115 file_type_suffix = ".bin";
116 break;
118 if (!google::protobuf::TextFormat::PrintToString(proto, &output_string)) {
119 LOG(WARNING) << "Printing to string failed.";
120 return false;
121 }
122 break;
124 google::protobuf::util::JsonPrintOptions options;
125 options.add_whitespace = true;
126 options.always_print_primitive_fields = true;
127 options.preserve_proto_field_names = true;
128 if (!google::protobuf::util::MessageToJsonString(proto, &output_string,
129 options)
130 .ok()) {
131 LOG(WARNING) << "Printing to stream failed.";
132 return false;
133 }
134 file_type_suffix = ".json";
135 break;
136 }
137 if (gzipped) {
138 std::string gzip_string;
139 GzipString(output_string, &gzip_string);
140 output_string.swap(gzip_string);
141 file_type_suffix += ".gz";
142 }
143 std::string output_filename(filename);
144 if (append_extension_to_file_name) output_filename += file_type_suffix;
145 VLOG(1) << "Writing " << output_string.size() << " bytes to "
146 << output_filename;
147 if (!file::SetContents(output_filename, output_string, file::Defaults())
148 .ok()) {
149 LOG(WARNING) << "Writing to file failed.";
150 return false;
151 }
152 return true;
153}
154
155} // namespace operations_research
#define CHECK_OK(x)
Definition: base/logging.h:40
#define LOG(severity)
Definition: base/logging.h:420
#define VLOG(verboselevel)
Definition: base/logging.h:978
CpModelProto proto
void GzipString(absl::string_view uncompressed, std::string *compressed)
Definition: gzipstring.h:63
bool GunzipString(const std::string &str, std::string *out)
Definition: gzipstring.h:22
const int WARNING
Definition: log_severity.h:31
absl::Status GetContents(const absl::string_view &filename, std::string *output, int flags)
Definition: file.cc:163
int Defaults()
Definition: base/file.h:119
absl::Status SetContents(const absl::string_view &filename, const absl::string_view &contents, int flags)
Definition: file.cc:196
The vehicle routing library lets one model and solve generic vehicle routing problems ranging from th...
absl::StatusOr< std::string > ReadFileToString(absl::string_view filename)
Definition: file_util.cc:32
bool WriteProtoToFile(absl::string_view filename, const google::protobuf::Message &proto, ProtoWriteFormat proto_write_format, bool gzipped, bool append_extension_to_file_name)
Definition: file_util.cc:102
bool ReadFileToProto(absl::string_view filename, google::protobuf::Message *proto)
Definition: file_util.cc:43
#define RETURN_IF_ERROR(expr)
Definition: status_macros.h:27