You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@allura.apache.org by br...@apache.org on 2024/10/02 16:06:50 UTC
(allura) branch db/8568 created (now 0df127ec2)
This is an automated email from the ASF dual-hosted git repository.
brondsem pushed a change to branch db/8568
in repository https://gitbox.apache.org/repos/asf/allura.git
at 0df127ec2 [#8568] support basic auth to solr
This branch includes the following new commits:
new 0df127ec2 [#8568] support basic auth to solr
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(allura) 01/01: [#8568] support basic auth to solr
Posted by br...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
brondsem pushed a commit to branch db/8568
in repository https://gitbox.apache.org/repos/asf/allura.git
commit 0df127ec26a12a22068338be9ce3e9355bf630de
Author: Dave Brondsema <db...@slashdotmedia.com>
AuthorDate: Wed Oct 2 11:52:41 2024 -0400
[#8568] support basic auth to solr
---
Allura/allura/command/show_models.py | 10 ++++++++--
Allura/allura/lib/app_globals.py | 13 +++++++++++--
Allura/allura/lib/solr.py | 22 ++++++++++++++++------
Allura/allura/tasks/index_tasks.py | 19 ++++++++++---------
Allura/allura/tests/test_commands.py | 4 ++--
Allura/allura/tests/unit/test_solr.py | 11 +++++++----
Allura/development.ini | 9 +++++++--
7 files changed, 61 insertions(+), 27 deletions(-)
diff --git a/Allura/allura/command/show_models.py b/Allura/allura/command/show_models.py
index 7bffebffa..b5d9d4300 100644
--- a/Allura/allura/command/show_models.py
+++ b/Allura/allura/command/show_models.py
@@ -79,6 +79,8 @@ class ReindexCommand(base.Command):
'which are needed for some markdown macros to run properly')
parser.add_option('--solr-hosts', dest='solr_hosts',
help='Override the solr host(s) to post to. Comma-separated list of solr server URLs')
+ parser.add_option('--solr-creds', dest='solr_creds',
+ help='Creds for the solr host(s). Comma-separated list of user:pwd strings')
parser.add_option(
'--max-chunk', dest='max_chunk', type=int, default=100 * 1000,
help='Max number of artifacts to index in one Solr update command')
@@ -148,9 +150,13 @@ class ReindexCommand(base.Command):
@property
def add_artifact_kwargs(self):
+ kwargs = {}
if self.options.solr_hosts:
- return {'solr_hosts': self.options.solr_hosts.split(',')}
- return {}
+ kwargs['solr_hosts'] = self.options.solr_hosts.split(',')
+ if self.options.solr_creds:
+ kwargs['solr_creds'] = [cred.split(':')
+ for cred in self.options.solr_creds.split(',')]
+ return kwargs
def _chunked_add_artifacts(self, ref_ids):
# ref_ids contains solr index ids which can easily be over
diff --git a/Allura/allura/lib/app_globals.py b/Allura/allura/lib/app_globals.py
index d4bf931a4..fc6bfbba9 100644
--- a/Allura/allura/lib/app_globals.py
+++ b/Allura/allura/lib/app_globals.py
@@ -203,13 +203,22 @@ class Globals:
self.solr_server = aslist(config.get('solr.server'), ',')
# skip empty strings in case of extra commas
self.solr_server = [s for s in self.solr_server if s]
+ push_auths = zip(aslist(config.get('solr.user'), ','),
+ aslist(config.get('solr.pass'), ','))
self.solr_query_server = config.get('solr.query_server')
+ query_auth = (config.get('solr.query_user'), config.get('solr.query_pass'))
if self.solr_server:
self.solr = make_solr_from_config(
- self.solr_server, self.solr_query_server)
+ self.solr_server, self.solr_query_server,
+ push_servers_auths=push_auths,
+ query_server_auth=query_auth,
+ )
self.solr_short_timeout = make_solr_from_config(
self.solr_server, self.solr_query_server,
- timeout=int(config.get('solr.short_timeout', 10)))
+ push_servers_auths=push_auths,
+ query_server_auth=query_auth,
+ timeout=int(config.get('solr.short_timeout', 10)),
+ )
else: # pragma no cover
log.warning('Solr config not set; using in-memory MockSOLR')
self.solr = self.solr_short_timeout = MockSOLR()
diff --git a/Allura/allura/lib/solr.py b/Allura/allura/lib/solr.py
index 7635b36d6..012fb728a 100644
--- a/Allura/allura/lib/solr.py
+++ b/Allura/allura/lib/solr.py
@@ -14,10 +14,12 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
+from __future__ import annotations
import json
import logging
+from itertools import zip_longest
+from collections.abc import Iterable
from tg import config
from webob.exc import HTTPRequestEntityTooLarge
@@ -61,7 +63,10 @@ def escape_solr_arg(term):
return term
-def make_solr_from_config(push_servers, query_server=None, **kwargs):
+def make_solr_from_config(push_servers: Iterable[str], query_server: str|None=None,
+ push_servers_auths: Iterable[tuple[str, str] | None] = (),
+ query_server_auth: tuple[str, str] | None = None,
+ **kwargs):
"""
Make a :class:`Solr <Solr>` instance from config defaults. Use
`**kwargs` to override any value
@@ -72,7 +77,7 @@ def make_solr_from_config(push_servers, query_server=None, **kwargs):
timeout=int(config.get('solr.long_timeout', 60)),
)
solr_kwargs.update(kwargs)
- return Solr(push_servers, query_server, **solr_kwargs)
+ return Solr(push_servers, query_server, push_servers_auths, query_server_auth, **solr_kwargs)
class Solr:
@@ -87,13 +92,18 @@ class Solr:
unless explicitly overridden.
"""
- def __init__(self, push_servers, query_server=None,
+ def __init__(self, push_servers: Iterable[str], query_server: str|None = None,
+ push_servers_auths: Iterable[tuple[str, str] | None] = (),
+ query_server_auth: tuple[str, str] | None = None,
commit=True, commitWithin=None, **kw):
- self.push_pool = [pysolr.Solr(s, **kw) for s in push_servers]
+ self.push_pool = [pysolr.Solr(s, auth=auth, **kw)
+ for s, auth in zip_longest(push_servers, push_servers_auths)]
if query_server:
- self.query_server = pysolr.Solr(query_server, **kw)
+ self.query_server = pysolr.Solr(query_server, auth=query_server_auth, **kw)
else:
self.query_server = self.push_pool[0]
+ if query_server_auth:
+ self.query_server.auth = query_server_auth
self._commit = commit
self.commitWithin = commitWithin
diff --git a/Allura/allura/tasks/index_tasks.py b/Allura/allura/tasks/index_tasks.py
index aec34c917..75c128f6f 100644
--- a/Allura/allura/tasks/index_tasks.py
+++ b/Allura/allura/tasks/index_tasks.py
@@ -18,6 +18,7 @@ from __future__ import annotations
import sys
import logging
+from collections.abc import Iterable
from contextlib import contextmanager
import typing
@@ -38,12 +39,12 @@ if typing.TYPE_CHECKING:
log = logging.getLogger(__name__)
-def __get_solr(solr_hosts=None):
- return make_solr_from_config(solr_hosts) if solr_hosts else g.solr
+def __get_solr(solr_hosts=None, solr_creds=()):
+ return make_solr_from_config(solr_hosts, push_servers_auths=solr_creds) if solr_hosts else g.solr
-def __add_objects(objects, solr_hosts=None):
- solr_instance = __get_solr(solr_hosts)
+def __add_objects(objects, solr_hosts=None, solr_creds=()):
+ solr_instance = __get_solr(solr_hosts, solr_creds)
solr_instance.add([obj.solarize() for obj in objects])
@@ -94,12 +95,12 @@ def del_users(user_solr_ids):
@task
-def add_artifacts(ref_ids, update_solr=True, update_refs=True, solr_hosts=None):
+def add_artifacts(ref_ids, update_solr=True, update_refs=True,
+ solr_hosts: Iterable[str] = (),
+ solr_creds: Iterable[tuple[str, str]] = (),
+ ):
'''
Add the referenced artifacts to SOLR and shortlinks.
-
- :param solr_hosts: a list of solr hosts to use instead of the defaults
- :type solr_hosts: [str]
'''
from allura import model as M
from allura.lib.search import find_shortlinks
@@ -146,7 +147,7 @@ def add_artifacts(ref_ids, update_solr=True, update_refs=True, solr_hosts=None):
log.info("Solr.add raised HTTPRequestEntityTooLarge but there is only one artifact. Raising exception.")
raise
- _add_artifact(__get_solr(solr_hosts), solr_updates)
+ _add_artifact(__get_solr(solr_hosts, solr_creds), solr_updates)
if len(exceptions) == 1:
raise exceptions[0][1].with_traceback(exceptions[0][2])
diff --git a/Allura/allura/tests/test_commands.py b/Allura/allura/tests/test_commands.py
index c228835af..994b44d33 100644
--- a/Allura/allura/tests/test_commands.py
+++ b/Allura/allura/tests/test_commands.py
@@ -532,7 +532,7 @@ class TestReindexCommand:
@patch('allura.command.show_models.add_artifacts')
def test_chunked_add_artifacts(self, add_artifacts):
cmd = show_models.ReindexCommand('reindex')
- cmd.options = Mock(tasks=True, max_chunk=10 * 1000, ming_config=None)
+ cmd.options = Mock(tasks=True, max_chunk=10 * 1000, ming_config=None, solr_creds='')
ref_ids = list(range(10 * 1000 * 2 + 20))
cmd._chunked_add_artifacts(ref_ids)
assert len(add_artifacts.post.call_args_list) == 3
@@ -575,7 +575,7 @@ class TestReindexCommand:
raise pymongo.errors.InvalidDocument("Cannot encode object...")
add_artifacts.post.side_effect = on_post
cmd = show_models.ReindexCommand('reindex')
- cmd.options = Mock(ming_config=None)
+ cmd.options = Mock(ming_config=None, solr_creds='')
with td.raises(pymongo.errors.InvalidDocument):
cmd._post_add_artifacts(list(range(5)))
diff --git a/Allura/allura/tests/unit/test_solr.py b/Allura/allura/tests/unit/test_solr.py
index 384a985bf..7dd7328e6 100644
--- a/Allura/allura/tests/unit/test_solr.py
+++ b/Allura/allura/tests/unit/test_solr.py
@@ -39,15 +39,18 @@ class TestSolr(unittest.TestCase):
@mock.patch('allura.lib.solr.pysolr')
def test_init(self, pysolr):
servers = ['server1', 'server2']
- solr = Solr(servers, commit=False, commitWithin='10000')
- calls = [mock.call('server1'), mock.call('server2')]
+ auths = [('u', 'pwd'),]
+ solr = Solr(servers, push_servers_auths=auths, commit=False, commitWithin='10000')
+ calls = [mock.call('server1', auth=('u', 'pwd')), mock.call('server2', auth=None)]
pysolr.Solr.assert_has_calls(calls)
assert len(solr.push_pool) == 2
pysolr.reset_mock()
solr = Solr(servers, 'server3', commit=False, commitWithin='10000')
- calls = [mock.call('server1'), mock.call('server2'),
- mock.call('server3')]
+ calls = [mock.call('server1', auth=None),
+ mock.call('server2', auth=None),
+ mock.call('server3', auth=None),
+ ]
pysolr.Solr.assert_has_calls(calls)
assert len(solr.push_pool) == 2
diff --git a/Allura/development.ini b/Allura/development.ini
index b91356088..9d64bd910 100644
--- a/Allura/development.ini
+++ b/Allura/development.ini
@@ -571,10 +571,15 @@ stats.sample_rate = 1
; number of seconds to sleep between checking for new tasks
monq.poll_interval=2
-; SOLR setup
+; SOLR setup. This can be a comma-separated list, to index into multiple locations
solr.server = http://localhost:8983/solr/allura
-; Alternate server to use just for querying
+# auth for solr, if needed. These can be a list too, to match solr.server
+;solr.user =
+;solr.pass =
+; Alternate server to use just for querying. Single server, not a list.
;solr.query_server =
+;solr.query_user =
+;solr.query_pass =
; Shorter timeout for search queries (longer timeout for saving to solr)
solr.short_timeout = 10
; commit on every add/delete?