Skip to content

Commit

Permalink
Get rid of usage on multiOpen in chinese addons (fcitx/fcitx5#1052)
Browse files Browse the repository at this point in the history
  • Loading branch information
@wengxt
wengxt committed May 25, 2024
1 parent ab4a95b commit 331fb64
Show file tree
Hide file tree
Showing 7 changed files with 162 additions and 125 deletions.
10 changes: 5 additions & 5 deletions gui/pinyindictmanager/filelistmodel.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,15 +76,15 @@ Qt::ItemFlags FileListModel::flags(const QModelIndex &index) const {
void FileListModel::loadFileList() {
beginResetModel();
fileList_.clear();
auto files = StandardPath::global().multiOpen(
StandardPath::Type::PkgData, "pinyin/dictionaries", O_RDONLY,
filter::Suffix(".dict"));
auto files = StandardPath::global().locate(StandardPath::Type::PkgData,
"pinyin/dictionaries",
filter::Suffix(".dict"));
std::map<std::string, bool> enableMap;
for (const auto &file : files) {
enableMap[file.first] = true;
}
auto disableFiles = StandardPath::global().multiOpen(
StandardPath::Type::PkgData, "pinyin/dictionaries", O_RDONLY,
auto disableFiles = StandardPath::global().locate(
StandardPath::Type::PkgData, "pinyin/dictionaries",
filter::Suffix(".dict.disable"));
for (const auto &file : disableFiles) {
// Remove .disable suffix.
Expand Down
113 changes: 55 additions & 58 deletions im/pinyin/pinyin.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@
#include <fcitx/text.h>
#include <fcitx/userinterface.h>
#include <fmt/core.h>
#include <fstream>
#include <functional>
#include <future>
#include <istream>
Expand Down Expand Up @@ -482,33 +483,35 @@ void PinyinEngine::updateUI(InputContext *inputContext) {
/// }}}

/// Create spell candidate {{{
int engNess;
auto [parsedPy, parsedPyCursor] = state->context_.preeditWithCursor(
libime::PinyinPreeditMode::RawText);
if (*config_.spellEnabled && spell() &&
parsedPyCursor >= selectedSentence.size() &&
selectedLength <= context.cursor() &&
(engNess = englishNess(parsedPy, context.useShuangpin()))) {
parsedPyCursor -= selectedSentence.length();
parsedPy =
parsedPy.substr(selectedSentence.size(),
parsedPyCursor > selectedSentence.length()
? parsedPyCursor - selectedSentence.length()
: std::string::npos);
auto results = spell()->call<ISpell::hintWithProvider>(
"en", SpellProvider::Custom, pyBeforeCursor, engNess);
std::string bestSentence;
if (!candidates.empty()) {
bestSentence = candidates[0].toString();
}
int position = 1;
for (auto &result : results) {
if (customCandidateSet.count(result) ||
context.candidatesToCursorSet().count(result)) {
continue;
selectedLength <= context.cursor()) {
int engNess = englishNess(parsedPy, context.useShuangpin());
if (engNess) {
parsedPyCursor -= selectedSentence.length();
parsedPy = parsedPy.substr(
selectedSentence.size(),
parsedPyCursor > selectedSentence.length()
? parsedPyCursor - selectedSentence.length()
: std::string::npos);
auto results = spell()->call<ISpell::hintWithProvider>(
"en", SpellProvider::Custom, pyBeforeCursor, engNess);
std::string bestSentence;
if (!candidates.empty()) {
bestSentence = candidates[0].toString();
}
int position = 1;
for (auto &result : results) {
if (customCandidateSet.count(result) ||
context.candidatesToCursorSet().count(result)) {
continue;
}
extraCandidates.push_back(
std::make_unique<SpellCandidateWord>(
this, result, pyBeforeCursor.size(), position++));
}
extraCandidates.push_back(std::make_unique<SpellCandidateWord>(
this, result, pyBeforeCursor.size(), position++));
}
}
/// }}}
Expand Down Expand Up @@ -818,35 +821,30 @@ void PinyinEngine::loadSymbols(const StandardPathFile &file) {
}
}

void PinyinEngine::loadDict(StandardPathFile file,
void PinyinEngine::loadDict(const std::string &fullPath,
std::list<std::unique_ptr<TaskToken>> &taskTokens) {
if (file.fd() < 0) {
if (fullPath.empty()) {
return;
}
ime_->dict()->addEmptyDict();
PINYIN_DEBUG() << "Loading pinyin dict " << file.path();
auto path = file.path();
std::packaged_task<libime::PinyinDictionary::TrieType()> task(
[file = std::move(file)]() {
boost::iostreams::stream_buffer<
boost::iostreams::file_descriptor_source>
buffer(file.fd(), boost::iostreams::file_descriptor_flags::
never_close_handle);
std::istream in(&buffer);
auto trie = libime::PinyinDictionary::load(
in, libime::PinyinDictFormat::Binary);
return trie;
});
PINYIN_DEBUG() << "Loading pinyin dict " << fullPath;
std::packaged_task<libime::PinyinDictionary::TrieType()> task([fullPath]() {
std::ifstream in(fullPath, std::ios::in | std::ios::binary);
auto trie = libime::PinyinDictionary::load(
in, libime::PinyinDictFormat::Binary);
return trie;
});
taskTokens.push_back(worker_.addTask(
std::move(task),
[this, index = ime_->dict()->dictSize() - 1,
path](std::shared_future<libime::PinyinDictionary::TrieType> &future) {
[this, index = ime_->dict()->dictSize() - 1, fullPath](
std::shared_future<libime::PinyinDictionary::TrieType> &future) {
try {
PINYIN_DEBUG() << "Load pinyin dict " << path << " finished.";
PINYIN_DEBUG()
<< "Load pinyin dict " << fullPath << " finished.";
ime_->dict()->setTrie(index, future.get());
} catch (const std::exception &e) {
PINYIN_ERROR() << "Failed to load pinyin dict " << path << ": "
<< e.what();
PINYIN_ERROR() << "Failed to load pinyin dict " << fullPath
<< ": " << e.what();
}
}));
}
Expand All @@ -859,20 +857,19 @@ void PinyinEngine::loadBuiltInDict() {
loadSymbols(file);
}
{
auto file = standardPath.open(StandardPath::Type::PkgData,
"pinyin/chaizi.dict", O_RDONLY);
loadDict(std::move(file), persistentTask_);
auto file = standardPath.locate(StandardPath::Type::PkgData,
"pinyin/chaizi.dict");
loadDict(file, persistentTask_);
}
{
auto file = standardPath.open(StandardPath::Type::Data,
"libime/extb.dict", O_RDONLY);
auto file =
standardPath.locate(StandardPath::Type::Data, "libime/extb.dict");
// Try again with absolute libime path.
if (!file.isValid()) {
file = standardPath.open(StandardPath::Type::Data,
LIBIME_INSTALL_PKGDATADIR "/extb.dict",
O_RDONLY);
if (file.empty()) {
file = standardPath.locate(StandardPath::Type::Data,
LIBIME_INSTALL_PKGDATADIR "/extb.dict");
}
loadDict(std::move(file), persistentTask_);
loadDict(file, persistentTask_);
}
if (ime_->dict()->dictSize() !=
libime::TrieDictionary::UserDict + 1 + NumBuiltInDict) {
Expand All @@ -882,12 +879,12 @@ void PinyinEngine::loadBuiltInDict() {

void PinyinEngine::loadExtraDict() {
const auto &standardPath = StandardPath::global();
auto files = standardPath.multiOpen(StandardPath::Type::PkgData,
"pinyin/dictionaries", O_RDONLY,
filter::Suffix(".dict"));
auto disableFiles = standardPath.multiOpen(StandardPath::Type::PkgData,
"pinyin/dictionaries", O_RDONLY,
filter::Suffix(".dict.disable"));
auto files =
standardPath.locate(StandardPath::Type::PkgData, "pinyin/dictionaries",
filter::Suffix(".dict"));
auto disableFiles =
standardPath.locate(StandardPath::Type::PkgData, "pinyin/dictionaries",
filter::Suffix(".dict.disable"));
FCITX_ASSERT(ime_->dict()->dictSize() >=
libime::TrieDictionary::UserDict + NumBuiltInDict + 1)
<< "Dict size: " << ime_->dict()->dictSize();
Expand Down
2 changes: 1 addition & 1 deletion im/pinyin/pinyin.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -427,7 +427,7 @@ class PinyinEngine final : public InputMethodEngineV3,
void loadExtraDict();
void loadCustomPhrase();
void loadSymbols(const StandardPathFile &file);
void loadDict(StandardPathFile file,
void loadDict(const std::string &fullPath,
std::list<std::unique_ptr<TaskToken>> &taskTokens);
void saveCustomPhrase();

Expand Down
27 changes: 19 additions & 8 deletions im/table/ime.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,15 +8,31 @@
#include <boost/iostreams/device/file_descriptor.hpp>
#include <boost/iostreams/stream_buffer.hpp>
#include <boost/range/adaptor/reversed.hpp>
#include <cstdint>
#include <exception>
#include <fcitx-config/iniparser.h>
#include <fcitx-config/rawconfig.h>
#include <fcitx-utils/key.h>
#include <fcitx-utils/log.h>
#include <fcitx-utils/macros.h>
#include <fcitx-utils/standardpath.h>
#include <fcitx-utils/stringutils.h>
#include <fcntl.h>
#include <fstream>
#include <istream>
#include <libime/core/languagemodel.h>
#include <libime/core/userlanguagemodel.h>
#include <libime/core/utils.h>
#include <libime/table/tablebaseddictionary.h>
#include <libime/table/tableoptions.h>
#include <memory>
#include <ostream>
#include <set>
#include <stdexcept>
#include <string>
#include <tuple>
#include <unordered_set>
#include <utility>

namespace fcitx {

Expand Down Expand Up @@ -156,18 +172,13 @@ TableIME::requestDict(const std::string &name) {
}

dict->removeAllExtra();
auto extraDicts = StandardPath::global().multiOpen(
auto extraDicts = StandardPath::global().locate(
StandardPath::Type::PkgData,
stringutils::concat("table/", name, ".dict.d"), O_RDONLY,
stringutils::concat("table/", name, ".dict.d"),
BinaryOrTextDict());
for (const auto &[name, file] : extraDicts) {
try {
boost::iostreams::stream_buffer<
boost::iostreams::file_descriptor_source>
buffer(file.fd(),
boost::iostreams::file_descriptor_flags::
never_close_handle);
std::istream in(&buffer);
std::ifstream in(file, std::ios::in | std::ios::binary);
const auto fileFormat = stringutils::endsWith(name, ".txt")
? libime::TableFormat::Text
: libime::TableFormat::Binary;
Expand Down
43 changes: 28 additions & 15 deletions modules/chttrans/chttrans.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,29 +6,46 @@
*/

#include "chttrans.h"
#include "chttrans-native.h"
#include "config.h"
#include <cassert>
#include <cstddef>
#include <exception>
#include <fcitx-config/iniparser.h>
#include <fcitx-utils/i18n.h>
#include <fcitx-utils/log.h>
#include <fcitx-utils/standardpath.h>
#include <fcitx-utils/stringutils.h>
#include <fcitx-utils/utf8.h>
#include <fcitx/addonfactory.h>
#include <fcitx/addoninstance.h>
#include <fcitx/addonmanager.h>
#include <fcitx/event.h>
#include <fcitx/inputcontext.h>
#include <fcitx/inputmethodentry.h>
#include <fcitx/instance.h>
#include <fcitx/text.h>
#include <fcitx/userinterface.h>
#include <fcitx/userinterfacemanager.h>
#include <fcntl.h>
#include <fmt/core.h>
#include <fmt/format.h>
#include <fstream>
#include <iterator>
#include <string>
#include <string_view>
#include <unordered_map>
#include <utility>
#include <vector>

#ifdef ENABLE_OPENCC
#include "chttrans-opencc.h"
#include <boost/iostreams/device/file_descriptor.hpp>
#include <boost/iostreams/stream.hpp>
#ifdef HAS_BOOST_JSON
#include <boost/json/parse.hpp>
#include <boost/json/src.hpp>
#include <boost/json/value_to.hpp>
#endif
#endif
#include "chttrans-native.h"
#include <fcitx/inputcontext.h>
#include <fcitx/userinterfacemanager.h>

using namespace fcitx;

Expand Down Expand Up @@ -112,7 +129,7 @@ Chttrans::Chttrans(fcitx::Instance *instance) : instance_(instance) {
outputFilterConn_ = instance_->connect<Instance::OutputFilter>(
[this](InputContext *inputContext, Text &text) {
// Short cut for empty string.
if (text.size() <= 0) {
if (text.empty()) {
return;
}
if (!toggleAction_.isParent(&inputContext->statusArea())) {
Expand Down Expand Up @@ -231,6 +248,7 @@ void Chttrans::populateConfig() {

void Chttrans::syncToConfig() {
std::vector<std::string> values_;
values_.reserve(enabledIM_.size());
for (const auto &id : enabledIM_) {
values_.push_back(id);
}
Expand All @@ -247,20 +265,15 @@ const Configuration *Chttrans::getConfig() const {
std::vector<std::pair<std::string, std::string>> profiles{
{"default", _("Default")}};
constexpr std::string_view JsonSuffix = ".json";
auto files = openCCStandardPath().multiOpen(
StandardPath::Type::PkgData, ".", O_RDONLY,
filter::Suffix(std::string(JsonSuffix)));
auto files =
openCCStandardPath().locate(StandardPath::Type::PkgData, ".",
filter::Suffix(std::string(JsonSuffix)));
profiles.reserve(files.size() + 1);
// files is std::map, so file name is already sorted.
for (const auto &file : files) {
#ifdef HAS_BOOST_JSON
try {
boost::iostreams::stream_buffer<
boost::iostreams::file_descriptor_source>
buffer(file.second.fd(),
boost::iostreams::file_descriptor_flags::
never_close_handle);
std::istream in(&buffer);
std::ifstream in(file.second, std::ios::in | std::ios::binary);
std::string strBuf(std::istreambuf_iterator<char>(in), {});
auto jv = boost::json::parse(strBuf);
const auto &obj = jv.as_object();
Expand Down
Loading

0 comments on commit 331fb64

Please sign in to comment.